We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add an option not to cleanup URLs for the Duplicate model
In the duplicate case, having two equal URL can be a strong indication that two bugs are the same.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bugbug/models/duplicate.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import random
7 
8 from sklearn.calibration import CalibratedClassifierCV
9 from sklearn.compose import ColumnTransformer
10 from sklearn.pipeline import Pipeline
11 from sklearn.preprocessing import LabelEncoder
12 from sklearn.svm import LinearSVC
13 
14 from bugbug import bug_features, bugzilla, feature_cleanup
15 from bugbug.model import BugCoupleModel
16 
17 REPORTERS_TO_IGNORE = {"intermittent-bug-filer@mozilla.bugs", "wptsync@mozilla.bugs"}
18 
19 
20 class LinearSVCWithLabelEncoding(CalibratedClassifierCV):
21     def __init__(self, clf):
22         super().__init__(clf)
23         self._le = LabelEncoder()
24 
25     def fit(self, X, y):
26         super().fit(X, y)
27         self._le.fit(y)
28 
29 
30 class DuplicateModel(BugCoupleModel):
31     def __init__(self, training_size=14000, lemmatization=False):
32         self.num_duplicates = training_size // 2
33         self.num_nondups_nondups = self.num_dup_nondups = training_size // 4
34 
35         BugCoupleModel.__init__(self, lemmatization)
36 
37         self.calculate_importance = False
38 
39         feature_extractors = [bug_features.is_same_product()]
40 
41         cleanup_functions = [
42             feature_cleanup.responses(),
43             feature_cleanup.hex(),
44             feature_cleanup.dll(),
45             feature_cleanup.fileref(),
46             feature_cleanup.url(),
47             feature_cleanup.synonyms(),
48             feature_cleanup.crash(),
49         ]
50 
51         self.extraction_pipeline = Pipeline(
52             [
53                 (
54                     "bug_extractor",
55                     bug_features.BugExtractor(feature_extractors, cleanup_functions),
56                 ),
57                 (
58                     "union",
59                     ColumnTransformer([("text", self.text_vectorizer(), "text")]),
60                 ),
61             ]
62         )
63 
64         self.clf = LinearSVCWithLabelEncoding(LinearSVC())
65 
66     def get_labels(self):
67 
68         random.seed(4)
69 
70         all_ids = set(
71             bug["id"]
72             for bug in bugzilla.get_bugs()
73             if bug["creator"] not in REPORTERS_TO_IGNORE
74             and "dupeme" not in bug["keywords"]
75         )
76 
77         classes = {}
78 
79         # Only store ids of bugs that have duplicates or are duplicates
80         duplicate_ids = []
81 
82         duplicates_num = 0
83         for bug_data in bugzilla.get_bugs():
84             bug_id = bug_data["id"]
85             if bug_id not in all_ids:
86                 continue
87 
88             if bug_data["dupe_of"] or len(bug_data["duplicates"]) > 0:
89                 duplicate_ids.append(bug_id)
90 
91             for duplicate_bug_id in bug_data["duplicates"]:
92                 if duplicate_bug_id not in all_ids:
93                     continue
94 
95                 duplicate_ids.append(duplicate_bug_id)
96 
97                 if duplicates_num < self.num_duplicates:
98                     classes[(bug_id, duplicate_bug_id)] = 1
99                 duplicates_num += 1
100 
101         # Remove duplicate duplicate IDs.
102         duplicate_ids = list(set(duplicate_ids))
103 
104         # Store all remaining ids
105         non_duplicate_ids = list(all_ids - set(duplicate_ids))
106 
107         print(f"Number of duplicate labels is: {self.num_duplicates}")
108 
109         # When the bug has no duplicates, we create dup-nondup labels.
110         dup_nondup_num = 0
111         while dup_nondup_num < self.num_dup_nondups:
112             bug_id1 = random.choice(duplicate_ids)
113             bug_id2 = random.choice(non_duplicate_ids)
114 
115             classes[(bug_id1, bug_id2)] = 0
116             dup_nondup_num += 1
117 
118         print(f"Number of hybrid labels is: {self.num_dup_nondups}")
119 
120         # Now we map non-dup to non-dup bug.
121         nondup_nondup_num = 0
122         while nondup_nondup_num < self.num_nondups_nondups:
123             bug_id1 = random.choice(non_duplicate_ids)
124             bug_id2 = random.choice(non_duplicate_ids)
125             if bug_id1 != bug_id2:
126                 classes[(bug_id1, bug_id2)] = 0
127                 nondup_nondup_num += 1
128 
129         print(f"Number of purely non-duplicate labels is: {self.num_nondups_nondups}")
130 
131         return classes, [0, 1]
132 
133     def get_feature_names(self):
134         return self.extraction_pipeline.named_steps["union"].get_feature_names()
135
```

Path: `run.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import argparse
7 import csv
8 import os
9 import sys
10 from datetime import datetime, timedelta
11 
12 import numpy as np
13 
14 from bugbug import bugzilla, db, repository
15 from bugbug.models import MODELS, get_model_class
16 
17 
18 def parse_args(args):
19     parser = argparse.ArgumentParser()
20     parser.add_argument(
21         "--lemmatization",
22         help="Perform lemmatization (using spaCy)",
23         action="store_true",
24     )
25     parser.add_argument(
26         "--training-set-size",
27         nargs="?",
28         default=14000,
29         type=int,
30         help="The size of the training set for the duplicate model",
31     )
32     parser.add_argument("--train", help="Perform training", action="store_true")
33     parser.add_argument(
34         "--goal", help="Goal of the classifier", choices=MODELS.keys(), default="defect"
35     )
36     parser.add_argument(
37         "--classifier",
38         help="Type of the classifier. Only used for component classification.",
39         choices=["default", "nn"],
40         default="default",
41     )
42     parser.add_argument("--classify", help="Perform evaluation", action="store_true")
43     parser.add_argument(
44         "--generate-sheet",
45         help="Perform evaluation on bugs from last week and generate a csv file",
46         action="store_true",
47     )
48     parser.add_argument("--token", help="Bugzilla token", action="store")
49     parser.add_argument(
50         "--historical",
51         help="""Analyze historical bugs. Only used for defect, bugtype,
52                 defectenhancementtask and regression tasks.""",
53         action="store_true",
54     )
55     return parser.parse_args(args)
56 
57 
58 def main(args):
59     model_file_name = "{}{}model".format(
60         args.goal, "" if args.classifier == "default" else args.classifier
61     )
62 
63     if args.goal == "component":
64         if args.classifier == "default":
65             model_class_name = "component"
66         else:
67             model_class_name = "component_nn"
68     else:
69         model_class_name = args.goal
70 
71     model_class = get_model_class(model_class_name)
72 
73     if args.train:
74         db.download(bugzilla.BUGS_DB)
75         db.download(repository.COMMITS_DB)
76 
77         historical_supported_tasks = [
78             "defect",
79             "bugtype",
80             "defectenhancementtask",
81             "regression",
82         ]
83 
84         if args.goal in historical_supported_tasks:
85             model = model_class(args.lemmatization, args.historical)
86         elif args.goal == "duplicate":
87             model = model_class(args.training_set_size, args.lemmatization)
88         else:
89             model = model_class(args.lemmatization)
90         model.train()
91     else:
92         model = model_class.load(model_file_name)
93 
94     if args.classify:
95         for bug in bugzilla.get_bugs():
96             print(
97                 f'https://bugzilla.mozilla.org/show_bug.cgi?id={ bug["id"] } - { bug["summary"]} '
98             )
99 
100             if model.calculate_importance:
101                 probas, importance = model.classify(
102                     bug, probabilities=True, importances=True
103                 )
104 
105                 feature_names = model.get_human_readable_feature_names()
106                 for i, (importance, index, is_positive) in enumerate(
107                     importance["importances"]
108                 ):
109                     print(
110                         f'{i + 1}. \'{feature_names[int(index)]}\' ({"+" if (is_positive) else "-"}{importance})'
111                     )
112             else:
113                 probas = model.classify(bug, probabilities=True, importances=False)
114 
115             if np.argmax(probas) == 1:
116                 print(f"Positive! {probas}")
117             else:
118                 print(f"Negative! {probas}")
119             input()
120 
121     if args.generate_sheet:
122         assert (
123             args.token is not None
124         ), "A Bugzilla token should be set in order to download bugs"
125         today = datetime.utcnow()
126         a_week_ago = today - timedelta(7)
127         bugzilla.set_token(args.token)
128         bug_ids = bugzilla.get_ids_between(a_week_ago, today)
129         bugs = bugzilla.get(bug_ids)
130 
131         print(f"Classifying {len(bugs)} bugs...")
132 
133         rows = [["Bug", f"{args.goal}(model)", args.goal, "Title"]]
134 
135         for bug in bugs.values():
136             p = model.classify(bug, probabilities=True)
137             rows.append(
138                 [
139                     f'https://bugzilla.mozilla.org/show_bug.cgi?id={bug["id"]}',
140                     "y" if p[0][1] >= 0.7 else "n",
141                     "",
142                     bug["summary"],
143                 ]
144             )
145 
146         os.makedirs("sheets", exist_ok=True)
147         with open(
148             os.path.join(
149                 "sheets",
150                 f'{args.goal}-{datetime.utcnow().strftime("%Y-%m-%d")}-labels.csv',
151             ),
152             "w",
153         ) as f:
154             writer = csv.writer(f)
155             writer.writerows(rows)
156 
157 
158 if __name__ == "__main__":
159     main(parse_args(sys.argv[1:]))
160
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bugbug/models/duplicate.py b/bugbug/models/duplicate.py
--- a/bugbug/models/duplicate.py
+++ b/bugbug/models/duplicate.py
@@ -28,7 +28,7 @@
 
 
 class DuplicateModel(BugCoupleModel):
-    def __init__(self, training_size=14000, lemmatization=False):
+    def __init__(self, training_size=14000, lemmatization=False, cleanup_urls=True):
         self.num_duplicates = training_size // 2
         self.num_nondups_nondups = self.num_dup_nondups = training_size // 4
 
@@ -43,11 +43,13 @@
             feature_cleanup.hex(),
             feature_cleanup.dll(),
             feature_cleanup.fileref(),
-            feature_cleanup.url(),
             feature_cleanup.synonyms(),
             feature_cleanup.crash(),
         ]
 
+        if cleanup_urls:
+            cleanup_functions.append(feature_cleanup.url())
+
         self.extraction_pipeline = Pipeline(
             [
                 (
diff --git a/run.py b/run.py
--- a/run.py
+++ b/run.py
@@ -29,6 +29,13 @@
         type=int,
         help="The size of the training set for the duplicate model",
     )
+    parser.add_argument(
+        "--disable-url-cleanup",
+        help="Don't cleanup urls when training the duplicate model",
+        dest="cleanup_urls",
+        default=True,
+        action="store_false",
+    )
     parser.add_argument("--train", help="Perform training", action="store_true")
     parser.add_argument(
         "--goal", help="Goal of the classifier", choices=MODELS.keys(), default="defect"
@@ -84,7 +91,9 @@
         if args.goal in historical_supported_tasks:
             model = model_class(args.lemmatization, args.historical)
         elif args.goal == "duplicate":
-            model = model_class(args.training_set_size, args.lemmatization)
+            model = model_class(
+                args.training_set_size, args.lemmatization, args.cleanup_urls
+            )
         else:
             model = model_class(args.lemmatization)
         model.train()

{"golden_diff": "diff --git a/bugbug/models/duplicate.py b/bugbug/models/duplicate.py\n--- a/bugbug/models/duplicate.py\n+++ b/bugbug/models/duplicate.py\n@@ -28,7 +28,7 @@\n \n \n class DuplicateModel(BugCoupleModel):\n-    def __init__(self, training_size=14000, lemmatization=False):\n+    def __init__(self, training_size=14000, lemmatization=False, cleanup_urls=True):\n         self.num_duplicates = training_size // 2\n         self.num_nondups_nondups = self.num_dup_nondups = training_size // 4\n \n@@ -43,11 +43,13 @@\n             feature_cleanup.hex(),\n             feature_cleanup.dll(),\n             feature_cleanup.fileref(),\n-            feature_cleanup.url(),\n             feature_cleanup.synonyms(),\n             feature_cleanup.crash(),\n         ]\n \n+        if cleanup_urls:\n+            cleanup_functions.append(feature_cleanup.url())\n+\n         self.extraction_pipeline = Pipeline(\n             [\n                 (\ndiff --git a/run.py b/run.py\n--- a/run.py\n+++ b/run.py\n@@ -29,6 +29,13 @@\n         type=int,\n         help=\"The size of the training set for the duplicate model\",\n     )\n+    parser.add_argument(\n+        \"--disable-url-cleanup\",\n+        help=\"Don't cleanup urls when training the duplicate model\",\n+        dest=\"cleanup_urls\",\n+        default=True,\n+        action=\"store_false\",\n+    )\n     parser.add_argument(\"--train\", help=\"Perform training\", action=\"store_true\")\n     parser.add_argument(\n         \"--goal\", help=\"Goal of the classifier\", choices=MODELS.keys(), default=\"defect\"\n@@ -84,7 +91,9 @@\n         if args.goal in historical_supported_tasks:\n             model = model_class(args.lemmatization, args.historical)\n         elif args.goal == \"duplicate\":\n-            model = model_class(args.training_set_size, args.lemmatization)\n+            model = model_class(\n+                args.training_set_size, args.lemmatization, args.cleanup_urls\n+            )\n         else:\n             model = model_class(args.lemmatization)\n         model.train()\n", "issue": "Add an option not to cleanup URLs for the Duplicate model\nIn the duplicate case, having two equal URL can be a strong indication that two bugs are the same.\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport random\n\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.svm import LinearSVC\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup\nfrom bugbug.model import BugCoupleModel\n\nREPORTERS_TO_IGNORE = {\"intermittent-bug-filer@mozilla.bugs\", \"wptsync@mozilla.bugs\"}\n\n\nclass LinearSVCWithLabelEncoding(CalibratedClassifierCV):\n    def __init__(self, clf):\n        super().__init__(clf)\n        self._le = LabelEncoder()\n\n    def fit(self, X, y):\n        super().fit(X, y)\n        self._le.fit(y)\n\n\nclass DuplicateModel(BugCoupleModel):\n    def __init__(self, training_size=14000, lemmatization=False):\n        self.num_duplicates = training_size // 2\n        self.num_nondups_nondups = self.num_dup_nondups = training_size // 4\n\n        BugCoupleModel.__init__(self, lemmatization)\n\n        self.calculate_importance = False\n\n        feature_extractors = [bug_features.is_same_product()]\n\n        cleanup_functions = [\n            feature_cleanup.responses(),\n            feature_cleanup.hex(),\n            feature_cleanup.dll(),\n            feature_cleanup.fileref(),\n            feature_cleanup.url(),\n            feature_cleanup.synonyms(),\n            feature_cleanup.crash(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(feature_extractors, cleanup_functions),\n                ),\n                (\n                    \"union\",\n                    ColumnTransformer([(\"text\", self.text_vectorizer(), \"text\")]),\n                ),\n            ]\n        )\n\n        self.clf = LinearSVCWithLabelEncoding(LinearSVC())\n\n    def get_labels(self):\n\n        random.seed(4)\n\n        all_ids = set(\n            bug[\"id\"]\n            for bug in bugzilla.get_bugs()\n            if bug[\"creator\"] not in REPORTERS_TO_IGNORE\n            and \"dupeme\" not in bug[\"keywords\"]\n        )\n\n        classes = {}\n\n        # Only store ids of bugs that have duplicates or are duplicates\n        duplicate_ids = []\n\n        duplicates_num = 0\n        for bug_data in bugzilla.get_bugs():\n            bug_id = bug_data[\"id\"]\n            if bug_id not in all_ids:\n                continue\n\n            if bug_data[\"dupe_of\"] or len(bug_data[\"duplicates\"]) > 0:\n                duplicate_ids.append(bug_id)\n\n            for duplicate_bug_id in bug_data[\"duplicates\"]:\n                if duplicate_bug_id not in all_ids:\n                    continue\n\n                duplicate_ids.append(duplicate_bug_id)\n\n                if duplicates_num < self.num_duplicates:\n                    classes[(bug_id, duplicate_bug_id)] = 1\n                duplicates_num += 1\n\n        # Remove duplicate duplicate IDs.\n        duplicate_ids = list(set(duplicate_ids))\n\n        # Store all remaining ids\n        non_duplicate_ids = list(all_ids - set(duplicate_ids))\n\n        print(f\"Number of duplicate labels is: {self.num_duplicates}\")\n\n        # When the bug has no duplicates, we create dup-nondup labels.\n        dup_nondup_num = 0\n        while dup_nondup_num < self.num_dup_nondups:\n            bug_id1 = random.choice(duplicate_ids)\n            bug_id2 = random.choice(non_duplicate_ids)\n\n            classes[(bug_id1, bug_id2)] = 0\n            dup_nondup_num += 1\n\n        print(f\"Number of hybrid labels is: {self.num_dup_nondups}\")\n\n        # Now we map non-dup to non-dup bug.\n        nondup_nondup_num = 0\n        while nondup_nondup_num < self.num_nondups_nondups:\n            bug_id1 = random.choice(non_duplicate_ids)\n            bug_id2 = random.choice(non_duplicate_ids)\n            if bug_id1 != bug_id2:\n                classes[(bug_id1, bug_id2)] = 0\n                nondup_nondup_num += 1\n\n        print(f\"Number of purely non-duplicate labels is: {self.num_nondups_nondups}\")\n\n        return classes, [0, 1]\n\n    def get_feature_names(self):\n        return self.extraction_pipeline.named_steps[\"union\"].get_feature_names()\n", "path": "bugbug/models/duplicate.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport argparse\nimport csv\nimport os\nimport sys\nfrom datetime import datetime, timedelta\n\nimport numpy as np\n\nfrom bugbug import bugzilla, db, repository\nfrom bugbug.models import MODELS, get_model_class\n\n\ndef parse_args(args):\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--lemmatization\",\n        help=\"Perform lemmatization (using spaCy)\",\n        action=\"store_true\",\n    )\n    parser.add_argument(\n        \"--training-set-size\",\n        nargs=\"?\",\n        default=14000,\n        type=int,\n        help=\"The size of the training set for the duplicate model\",\n    )\n    parser.add_argument(\"--train\", help=\"Perform training\", action=\"store_true\")\n    parser.add_argument(\n        \"--goal\", help=\"Goal of the classifier\", choices=MODELS.keys(), default=\"defect\"\n    )\n    parser.add_argument(\n        \"--classifier\",\n        help=\"Type of the classifier. Only used for component classification.\",\n        choices=[\"default\", \"nn\"],\n        default=\"default\",\n    )\n    parser.add_argument(\"--classify\", help=\"Perform evaluation\", action=\"store_true\")\n    parser.add_argument(\n        \"--generate-sheet\",\n        help=\"Perform evaluation on bugs from last week and generate a csv file\",\n        action=\"store_true\",\n    )\n    parser.add_argument(\"--token\", help=\"Bugzilla token\", action=\"store\")\n    parser.add_argument(\n        \"--historical\",\n        help=\"\"\"Analyze historical bugs. Only used for defect, bugtype,\n                defectenhancementtask and regression tasks.\"\"\",\n        action=\"store_true\",\n    )\n    return parser.parse_args(args)\n\n\ndef main(args):\n    model_file_name = \"{}{}model\".format(\n        args.goal, \"\" if args.classifier == \"default\" else args.classifier\n    )\n\n    if args.goal == \"component\":\n        if args.classifier == \"default\":\n            model_class_name = \"component\"\n        else:\n            model_class_name = \"component_nn\"\n    else:\n        model_class_name = args.goal\n\n    model_class = get_model_class(model_class_name)\n\n    if args.train:\n        db.download(bugzilla.BUGS_DB)\n        db.download(repository.COMMITS_DB)\n\n        historical_supported_tasks = [\n            \"defect\",\n            \"bugtype\",\n            \"defectenhancementtask\",\n            \"regression\",\n        ]\n\n        if args.goal in historical_supported_tasks:\n            model = model_class(args.lemmatization, args.historical)\n        elif args.goal == \"duplicate\":\n            model = model_class(args.training_set_size, args.lemmatization)\n        else:\n            model = model_class(args.lemmatization)\n        model.train()\n    else:\n        model = model_class.load(model_file_name)\n\n    if args.classify:\n        for bug in bugzilla.get_bugs():\n            print(\n                f'https://bugzilla.mozilla.org/show_bug.cgi?id={ bug[\"id\"] } - { bug[\"summary\"]} '\n            )\n\n            if model.calculate_importance:\n                probas, importance = model.classify(\n                    bug, probabilities=True, importances=True\n                )\n\n                feature_names = model.get_human_readable_feature_names()\n                for i, (importance, index, is_positive) in enumerate(\n                    importance[\"importances\"]\n                ):\n                    print(\n                        f'{i + 1}. \\'{feature_names[int(index)]}\\' ({\"+\" if (is_positive) else \"-\"}{importance})'\n                    )\n            else:\n                probas = model.classify(bug, probabilities=True, importances=False)\n\n            if np.argmax(probas) == 1:\n                print(f\"Positive! {probas}\")\n            else:\n                print(f\"Negative! {probas}\")\n            input()\n\n    if args.generate_sheet:\n        assert (\n            args.token is not None\n        ), \"A Bugzilla token should be set in order to download bugs\"\n        today = datetime.utcnow()\n        a_week_ago = today - timedelta(7)\n        bugzilla.set_token(args.token)\n        bug_ids = bugzilla.get_ids_between(a_week_ago, today)\n        bugs = bugzilla.get(bug_ids)\n\n        print(f\"Classifying {len(bugs)} bugs...\")\n\n        rows = [[\"Bug\", f\"{args.goal}(model)\", args.goal, \"Title\"]]\n\n        for bug in bugs.values():\n            p = model.classify(bug, probabilities=True)\n            rows.append(\n                [\n                    f'https://bugzilla.mozilla.org/show_bug.cgi?id={bug[\"id\"]}',\n                    \"y\" if p[0][1] >= 0.7 else \"n\",\n                    \"\",\n                    bug[\"summary\"],\n                ]\n            )\n\n        os.makedirs(\"sheets\", exist_ok=True)\n        with open(\n            os.path.join(\n                \"sheets\",\n                f'{args.goal}-{datetime.utcnow().strftime(\"%Y-%m-%d\")}-labels.csv',\n            ),\n            \"w\",\n        ) as f:\n            writer = csv.writer(f)\n            writer.writerows(rows)\n\n\nif __name__ == \"__main__\":\n    main(parse_args(sys.argv[1:]))\n", "path": "run.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport random\n\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.svm import LinearSVC\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup\nfrom bugbug.model import BugCoupleModel\n\nREPORTERS_TO_IGNORE = {\"intermittent-bug-filer@mozilla.bugs\", \"wptsync@mozilla.bugs\"}\n\n\nclass LinearSVCWithLabelEncoding(CalibratedClassifierCV):\n    def __init__(self, clf):\n        super().__init__(clf)\n        self._le = LabelEncoder()\n\n    def fit(self, X, y):\n        super().fit(X, y)\n        self._le.fit(y)\n\n\nclass DuplicateModel(BugCoupleModel):\n    def __init__(self, training_size=14000, lemmatization=False, cleanup_urls=True):\n        self.num_duplicates = training_size // 2\n        self.num_nondups_nondups = self.num_dup_nondups = training_size // 4\n\n        BugCoupleModel.__init__(self, lemmatization)\n\n        self.calculate_importance = False\n\n        feature_extractors = [bug_features.is_same_product()]\n\n        cleanup_functions = [\n            feature_cleanup.responses(),\n            feature_cleanup.hex(),\n            feature_cleanup.dll(),\n            feature_cleanup.fileref(),\n            feature_cleanup.synonyms(),\n            feature_cleanup.crash(),\n        ]\n\n        if cleanup_urls:\n            cleanup_functions.append(feature_cleanup.url())\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(feature_extractors, cleanup_functions),\n                ),\n                (\n                    \"union\",\n                    ColumnTransformer([(\"text\", self.text_vectorizer(), \"text\")]),\n                ),\n            ]\n        )\n\n        self.clf = LinearSVCWithLabelEncoding(LinearSVC())\n\n    def get_labels(self):\n\n        random.seed(4)\n\n        all_ids = set(\n            bug[\"id\"]\n            for bug in bugzilla.get_bugs()\n            if bug[\"creator\"] not in REPORTERS_TO_IGNORE\n            and \"dupeme\" not in bug[\"keywords\"]\n        )\n\n        classes = {}\n\n        # Only store ids of bugs that have duplicates or are duplicates\n        duplicate_ids = []\n\n        duplicates_num = 0\n        for bug_data in bugzilla.get_bugs():\n            bug_id = bug_data[\"id\"]\n            if bug_id not in all_ids:\n                continue\n\n            if bug_data[\"dupe_of\"] or len(bug_data[\"duplicates\"]) > 0:\n                duplicate_ids.append(bug_id)\n\n            for duplicate_bug_id in bug_data[\"duplicates\"]:\n                if duplicate_bug_id not in all_ids:\n                    continue\n\n                duplicate_ids.append(duplicate_bug_id)\n\n                if duplicates_num < self.num_duplicates:\n                    classes[(bug_id, duplicate_bug_id)] = 1\n                duplicates_num += 1\n\n        # Remove duplicate duplicate IDs.\n        duplicate_ids = list(set(duplicate_ids))\n\n        # Store all remaining ids\n        non_duplicate_ids = list(all_ids - set(duplicate_ids))\n\n        print(f\"Number of duplicate labels is: {self.num_duplicates}\")\n\n        # When the bug has no duplicates, we create dup-nondup labels.\n        dup_nondup_num = 0\n        while dup_nondup_num < self.num_dup_nondups:\n            bug_id1 = random.choice(duplicate_ids)\n            bug_id2 = random.choice(non_duplicate_ids)\n\n            classes[(bug_id1, bug_id2)] = 0\n            dup_nondup_num += 1\n\n        print(f\"Number of hybrid labels is: {self.num_dup_nondups}\")\n\n        # Now we map non-dup to non-dup bug.\n        nondup_nondup_num = 0\n        while nondup_nondup_num < self.num_nondups_nondups:\n            bug_id1 = random.choice(non_duplicate_ids)\n            bug_id2 = random.choice(non_duplicate_ids)\n            if bug_id1 != bug_id2:\n                classes[(bug_id1, bug_id2)] = 0\n                nondup_nondup_num += 1\n\n        print(f\"Number of purely non-duplicate labels is: {self.num_nondups_nondups}\")\n\n        return classes, [0, 1]\n\n    def get_feature_names(self):\n        return self.extraction_pipeline.named_steps[\"union\"].get_feature_names()\n", "path": "bugbug/models/duplicate.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport argparse\nimport csv\nimport os\nimport sys\nfrom datetime import datetime, timedelta\n\nimport numpy as np\n\nfrom bugbug import bugzilla, db, repository\nfrom bugbug.models import MODELS, get_model_class\n\n\ndef parse_args(args):\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--lemmatization\",\n        help=\"Perform lemmatization (using spaCy)\",\n        action=\"store_true\",\n    )\n    parser.add_argument(\n        \"--training-set-size\",\n        nargs=\"?\",\n        default=14000,\n        type=int,\n        help=\"The size of the training set for the duplicate model\",\n    )\n    parser.add_argument(\n        \"--disable-url-cleanup\",\n        help=\"Don't cleanup urls when training the duplicate model\",\n        dest=\"cleanup_urls\",\n        default=True,\n        action=\"store_false\",\n    )\n    parser.add_argument(\"--train\", help=\"Perform training\", action=\"store_true\")\n    parser.add_argument(\n        \"--goal\", help=\"Goal of the classifier\", choices=MODELS.keys(), default=\"defect\"\n    )\n    parser.add_argument(\n        \"--classifier\",\n        help=\"Type of the classifier. Only used for component classification.\",\n        choices=[\"default\", \"nn\"],\n        default=\"default\",\n    )\n    parser.add_argument(\"--classify\", help=\"Perform evaluation\", action=\"store_true\")\n    parser.add_argument(\n        \"--generate-sheet\",\n        help=\"Perform evaluation on bugs from last week and generate a csv file\",\n        action=\"store_true\",\n    )\n    parser.add_argument(\"--token\", help=\"Bugzilla token\", action=\"store\")\n    parser.add_argument(\n        \"--historical\",\n        help=\"\"\"Analyze historical bugs. Only used for defect, bugtype,\n                defectenhancementtask and regression tasks.\"\"\",\n        action=\"store_true\",\n    )\n    return parser.parse_args(args)\n\n\ndef main(args):\n    model_file_name = \"{}{}model\".format(\n        args.goal, \"\" if args.classifier == \"default\" else args.classifier\n    )\n\n    if args.goal == \"component\":\n        if args.classifier == \"default\":\n            model_class_name = \"component\"\n        else:\n            model_class_name = \"component_nn\"\n    else:\n        model_class_name = args.goal\n\n    model_class = get_model_class(model_class_name)\n\n    if args.train:\n        db.download(bugzilla.BUGS_DB)\n        db.download(repository.COMMITS_DB)\n\n        historical_supported_tasks = [\n            \"defect\",\n            \"bugtype\",\n            \"defectenhancementtask\",\n            \"regression\",\n        ]\n\n        if args.goal in historical_supported_tasks:\n            model = model_class(args.lemmatization, args.historical)\n        elif args.goal == \"duplicate\":\n            model = model_class(\n                args.training_set_size, args.lemmatization, args.cleanup_urls\n            )\n        else:\n            model = model_class(args.lemmatization)\n        model.train()\n    else:\n        model = model_class.load(model_file_name)\n\n    if args.classify:\n        for bug in bugzilla.get_bugs():\n            print(\n                f'https://bugzilla.mozilla.org/show_bug.cgi?id={ bug[\"id\"] } - { bug[\"summary\"]} '\n            )\n\n            if model.calculate_importance:\n                probas, importance = model.classify(\n                    bug, probabilities=True, importances=True\n                )\n\n                feature_names = model.get_human_readable_feature_names()\n                for i, (importance, index, is_positive) in enumerate(\n                    importance[\"importances\"]\n                ):\n                    print(\n                        f'{i + 1}. \\'{feature_names[int(index)]}\\' ({\"+\" if (is_positive) else \"-\"}{importance})'\n                    )\n            else:\n                probas = model.classify(bug, probabilities=True, importances=False)\n\n            if np.argmax(probas) == 1:\n                print(f\"Positive! {probas}\")\n            else:\n                print(f\"Negative! {probas}\")\n            input()\n\n    if args.generate_sheet:\n        assert (\n            args.token is not None\n        ), \"A Bugzilla token should be set in order to download bugs\"\n        today = datetime.utcnow()\n        a_week_ago = today - timedelta(7)\n        bugzilla.set_token(args.token)\n        bug_ids = bugzilla.get_ids_between(a_week_ago, today)\n        bugs = bugzilla.get(bug_ids)\n\n        print(f\"Classifying {len(bugs)} bugs...\")\n\n        rows = [[\"Bug\", f\"{args.goal}(model)\", args.goal, \"Title\"]]\n\n        for bug in bugs.values():\n            p = model.classify(bug, probabilities=True)\n            rows.append(\n                [\n                    f'https://bugzilla.mozilla.org/show_bug.cgi?id={bug[\"id\"]}',\n                    \"y\" if p[0][1] >= 0.7 else \"n\",\n                    \"\",\n                    bug[\"summary\"],\n                ]\n            )\n\n        os.makedirs(\"sheets\", exist_ok=True)\n        with open(\n            os.path.join(\n                \"sheets\",\n                f'{args.goal}-{datetime.utcnow().strftime(\"%Y-%m-%d\")}-labels.csv',\n            ),\n            \"w\",\n        ) as f:\n            writer = csv.writer(f)\n            writer.writerows(rows)\n\n\nif __name__ == \"__main__\":\n    main(parse_args(sys.argv[1:]))\n", "path": "run.py"}]}