We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
macro_avg_f_score bug
In the file flair/training_utils.py, when calculating macro_avg_f_score, I think 'recall' method should be called but 'precision' is called.

![image](https://user-images.githubusercontent.com/44130753/51589926-4b59bf00-1f23-11e9-964a-24ac0bc82589.png)

i think it needs to write like this:
`def macro_avg_f_score(self): 
         class_precisions = [self.precision(class_name) for class_name in self.get_classes()] 
         class_recalls = [self.recall(class_name) for class_name in self.get_classes()]`



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `flair/training_utils.py`
Content:
```
1 import itertools
2 import random
3 import logging
4 from collections import defaultdict
5 from enum import Enum
6 from pathlib import Path
7 from typing import List
8 from flair.data import Dictionary, Sentence
9 from functools import reduce
10 
11 
12 class Metric(object):
13 
14     def __init__(self, name):
15         self.name = name
16 
17         self._tps = defaultdict(int)
18         self._fps = defaultdict(int)
19         self._tns = defaultdict(int)
20         self._fns = defaultdict(int)
21 
22     def add_tp(self, class_name):
23         self._tps[class_name] += 1
24 
25     def add_tn(self, class_name):
26         self._tns[class_name] += 1
27 
28     def add_fp(self, class_name):
29         self._fps[class_name] += 1
30 
31     def add_fn(self, class_name):
32         self._fns[class_name] += 1
33 
34     def get_tp(self, class_name=None):
35         if class_name is None:
36             return sum([self._tps[class_name] for class_name in self.get_classes()])
37         return self._tps[class_name]
38 
39     def get_tn(self, class_name):
40         if class_name is None:
41             return sum([self._tns[class_name] for class_name in self.get_classes()])
42         return self._tns[class_name]
43 
44     def get_fp(self, class_name):
45         if class_name is None:
46             return sum([self._fps[class_name] for class_name in self.get_classes()])
47         return self._fps[class_name]
48 
49     def get_fn(self, class_name):
50         if class_name is None:
51             return sum([self._fns[class_name] for class_name in self.get_classes()])
52         return self._fns[class_name]
53 
54     def precision(self, class_name=None):
55         if self.get_tp(class_name) + self.get_fp(class_name) > 0:
56             return round(self.get_tp(class_name) / (self.get_tp(class_name) + self.get_fp(class_name)), 4)
57         return 0.0
58 
59     def recall(self, class_name=None):
60         if self.get_tp(class_name) + self.get_fn(class_name) > 0:
61             return round(self.get_tp(class_name) / (self.get_tp(class_name) + self.get_fn(class_name)), 4)
62         return 0.0
63 
64     def f_score(self, class_name=None):
65         if self.precision(class_name) + self.recall(class_name) > 0:
66             return round(2 * (self.precision(class_name) * self.recall(class_name))
67                          / (self.precision(class_name) + self.recall(class_name)), 4)
68         return 0.0
69 
70     def accuracy(self, class_name=None):
71         if self.get_tp(class_name) + self.get_tn(class_name) + self.get_fp(class_name) + self.get_fn(class_name) > 0:
72             return round(
73                 (self.get_tp(class_name) + self.get_tn(class_name))
74                 / (self.get_tp(class_name) + self.get_tn(class_name) + self.get_fp(class_name) + self.get_fn(class_name)),
75                 4)
76         return 0.0
77 
78     def micro_avg_f_score(self):
79         return self.f_score(None)
80 
81     def macro_avg_f_score(self):
82         class_precisions = [self.precision(class_name) for class_name in self.get_classes()]
83         class_recalls = [self.precision(class_name) for class_name in self.get_classes()]
84 
85         if len(class_precisions) > 0 and len(class_recalls) > 0:
86             macro_precision = sum(class_precisions) / len(class_precisions)
87             macro_recall = sum(class_recalls) / len(class_recalls)
88 
89             if macro_precision + macro_recall > 0:
90                 return round(2 * (macro_precision * macro_recall) / (macro_precision + macro_recall), 4)
91 
92         return 0.0
93 
94     def micro_avg_accuracy(self):
95         return self.accuracy(None)
96 
97     def macro_avg_accuracy(self):
98         class_accuracy = [self.accuracy(class_name) for class_name in self.get_classes()]
99 
100         if len(class_accuracy) > 0:
101             return round(sum(class_accuracy) / len(class_accuracy), 4)
102 
103         return 0.0
104 
105     def get_classes(self) -> List:
106         all_classes = set(itertools.chain(*[list(keys) for keys
107                                             in [self._tps.keys(), self._fps.keys(), self._tns.keys(),
108                                                 self._fns.keys()]]))
109         all_classes = [class_name for class_name in all_classes if class_name is not None]
110         all_classes.sort()
111         return all_classes
112 
113     def to_tsv(self):
114         return '{}\t{}\t{}\t{}'.format(
115             self.precision(),
116             self.recall(),
117             self.accuracy(),
118             self.micro_avg_f_score(),
119         )
120 
121     @staticmethod
122     def tsv_header(prefix=None):
123         if prefix:
124             return '{0}_PRECISION\t{0}_RECALL\t{0}_ACCURACY\t{0}_F-SCORE'.format(
125                 prefix)
126 
127         return 'PRECISION\tRECALL\tACCURACY\tF-SCORE'
128 
129     @staticmethod
130     def to_empty_tsv():
131         return '\t_\t_\t_\t_'
132 
133     def __str__(self):
134         all_classes = self.get_classes()
135         all_classes = [None] + all_classes
136         all_lines = [
137             '{0:<10}\ttp: {1} - fp: {2} - fn: {3} - tn: {4} - precision: {5:.4f} - recall: {6:.4f} - accuracy: {7:.4f} - f1-score: {8:.4f}'.format(
138                 self.name if class_name is None else class_name,
139                 self.get_tp(class_name), self.get_fp(class_name), self.get_fn(class_name), self.get_tn(class_name),
140                 self.precision(class_name), self.recall(class_name), self.accuracy(class_name),
141                 self.f_score(class_name))
142             for class_name in all_classes]
143         return '\n'.join(all_lines)
144 
145 
146 class EvaluationMetric(Enum):
147     MICRO_ACCURACY = 'micro-average accuracy'
148     MICRO_F1_SCORE = 'micro-average f1-score'
149     MACRO_ACCURACY = 'macro-average accuracy'
150     MACRO_F1_SCORE = 'macro-average f1-score'
151 
152 
153 class WeightExtractor(object):
154 
155     def __init__(self, directory: Path, number_of_weights: int = 10):
156         self.weights_file = init_output_file(directory, 'weights.txt')
157         self.weights_dict = defaultdict(lambda: defaultdict(lambda: list()))
158         self.number_of_weights = number_of_weights
159 
160     def extract_weights(self, state_dict, iteration):
161         for key in state_dict.keys():
162 
163             vec = state_dict[key]
164             weights_to_watch = min(self.number_of_weights, reduce(lambda x, y: x * y, list(vec.size())))
165 
166             if key not in self.weights_dict:
167                 self._init_weights_index(key, state_dict, weights_to_watch)
168 
169             for i in range(weights_to_watch):
170                 vec = state_dict[key]
171                 for index in self.weights_dict[key][i]:
172                     vec = vec[index]
173 
174                 value = vec.item()
175 
176                 with open(self.weights_file, 'a') as f:
177                     f.write('{}\t{}\t{}\t{}\n'.format(iteration, key, i, float(value)))
178 
179     def _init_weights_index(self, key, state_dict, weights_to_watch):
180         indices = {}
181 
182         i = 0
183         while len(indices) < weights_to_watch:
184             vec = state_dict[key]
185             cur_indices = []
186 
187             for x in range(len(vec.size())):
188                 index = random.randint(0, len(vec) - 1)
189                 vec = vec[index]
190                 cur_indices.append(index)
191 
192             if cur_indices not in list(indices.values()):
193                 indices[i] = cur_indices
194                 i += 1
195 
196         self.weights_dict[key] = indices
197 
198 
199 def clear_embeddings(sentences: List[Sentence], also_clear_word_embeddings=False):
200     """
201     Clears the embeddings from all given sentences.
202     :param sentences: list of sentences
203     """
204     for sentence in sentences:
205         sentence.clear_embeddings(also_clear_word_embeddings=also_clear_word_embeddings)
206 
207 
208 def init_output_file(base_path: Path, file_name: str) -> Path:
209     """
210     Creates a local file.
211     :param base_path: the path to the directory
212     :param file_name: the file name
213     :return: the created file
214     """
215     base_path.mkdir(parents=True, exist_ok=True)
216 
217     file = base_path / file_name
218     open(file, "w", encoding='utf-8').close()
219     return file
220 
221 
222 def convert_labels_to_one_hot(label_list: List[List[str]], label_dict: Dictionary) -> List[List[int]]:
223     """
224     Convert list of labels (strings) to a one hot list.
225     :param label_list: list of labels
226     :param label_dict: label dictionary
227     :return: converted label list
228     """
229     return [[1 if l in labels else 0 for l in label_dict.get_items()] for labels in label_list]
230 
231 
232 def log_line(log):
233     log.info('-' * 100)
234 
235 
236 def add_file_handler(log, output_file):
237     init_output_file(output_file.parents[0], output_file.name)
238     fh = logging.FileHandler(output_file)
239     fh.setLevel(logging.INFO)
240     formatter = logging.Formatter('%(asctime)-15s %(message)s')
241     fh.setFormatter(formatter)
242     log.addHandler(fh)
243
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/flair/training_utils.py b/flair/training_utils.py
--- a/flair/training_utils.py
+++ b/flair/training_utils.py
@@ -80,7 +80,7 @@
 
     def macro_avg_f_score(self):
         class_precisions = [self.precision(class_name) for class_name in self.get_classes()]
-        class_recalls = [self.precision(class_name) for class_name in self.get_classes()]
+        class_recalls = [self.recall(class_name) for class_name in self.get_classes()]
 
         if len(class_precisions) > 0 and len(class_recalls) > 0:
             macro_precision = sum(class_precisions) / len(class_precisions)

{"golden_diff": "diff --git a/flair/training_utils.py b/flair/training_utils.py\n--- a/flair/training_utils.py\n+++ b/flair/training_utils.py\n@@ -80,7 +80,7 @@\n \n     def macro_avg_f_score(self):\n         class_precisions = [self.precision(class_name) for class_name in self.get_classes()]\n-        class_recalls = [self.precision(class_name) for class_name in self.get_classes()]\n+        class_recalls = [self.recall(class_name) for class_name in self.get_classes()]\n \n         if len(class_precisions) > 0 and len(class_recalls) > 0:\n             macro_precision = sum(class_precisions) / len(class_precisions)\n", "issue": "macro_avg_f_score bug\nIn the file flair/training_utils.py, when calculating macro_avg_f_score, I think 'recall' method should be called but 'precision' is called.\r\n\r\n![image](https://user-images.githubusercontent.com/44130753/51589926-4b59bf00-1f23-11e9-964a-24ac0bc82589.png)\r\n\r\ni think it needs to write like this:\r\n`def macro_avg_f_score(self): \r\n         class_precisions = [self.precision(class_name) for class_name in self.get_classes()] \r\n         class_recalls = [self.recall(class_name) for class_name in self.get_classes()]`\r\n\r\n\n", "before_files": [{"content": "import itertools\nimport random\nimport logging\nfrom collections import defaultdict\nfrom enum import Enum\nfrom pathlib import Path\nfrom typing import List\nfrom flair.data import Dictionary, Sentence\nfrom functools import reduce\n\n\nclass Metric(object):\n\n    def __init__(self, name):\n        self.name = name\n\n        self._tps = defaultdict(int)\n        self._fps = defaultdict(int)\n        self._tns = defaultdict(int)\n        self._fns = defaultdict(int)\n\n    def add_tp(self, class_name):\n        self._tps[class_name] += 1\n\n    def add_tn(self, class_name):\n        self._tns[class_name] += 1\n\n    def add_fp(self, class_name):\n        self._fps[class_name] += 1\n\n    def add_fn(self, class_name):\n        self._fns[class_name] += 1\n\n    def get_tp(self, class_name=None):\n        if class_name is None:\n            return sum([self._tps[class_name] for class_name in self.get_classes()])\n        return self._tps[class_name]\n\n    def get_tn(self, class_name):\n        if class_name is None:\n            return sum([self._tns[class_name] for class_name in self.get_classes()])\n        return self._tns[class_name]\n\n    def get_fp(self, class_name):\n        if class_name is None:\n            return sum([self._fps[class_name] for class_name in self.get_classes()])\n        return self._fps[class_name]\n\n    def get_fn(self, class_name):\n        if class_name is None:\n            return sum([self._fns[class_name] for class_name in self.get_classes()])\n        return self._fns[class_name]\n\n    def precision(self, class_name=None):\n        if self.get_tp(class_name) + self.get_fp(class_name) > 0:\n            return round(self.get_tp(class_name) / (self.get_tp(class_name) + self.get_fp(class_name)), 4)\n        return 0.0\n\n    def recall(self, class_name=None):\n        if self.get_tp(class_name) + self.get_fn(class_name) > 0:\n            return round(self.get_tp(class_name) / (self.get_tp(class_name) + self.get_fn(class_name)), 4)\n        return 0.0\n\n    def f_score(self, class_name=None):\n        if self.precision(class_name) + self.recall(class_name) > 0:\n            return round(2 * (self.precision(class_name) * self.recall(class_name))\n                         / (self.precision(class_name) + self.recall(class_name)), 4)\n        return 0.0\n\n    def accuracy(self, class_name=None):\n        if self.get_tp(class_name) + self.get_tn(class_name) + self.get_fp(class_name) + self.get_fn(class_name) > 0:\n            return round(\n                (self.get_tp(class_name) + self.get_tn(class_name))\n                / (self.get_tp(class_name) + self.get_tn(class_name) + self.get_fp(class_name) + self.get_fn(class_name)),\n                4)\n        return 0.0\n\n    def micro_avg_f_score(self):\n        return self.f_score(None)\n\n    def macro_avg_f_score(self):\n        class_precisions = [self.precision(class_name) for class_name in self.get_classes()]\n        class_recalls = [self.precision(class_name) for class_name in self.get_classes()]\n\n        if len(class_precisions) > 0 and len(class_recalls) > 0:\n            macro_precision = sum(class_precisions) / len(class_precisions)\n            macro_recall = sum(class_recalls) / len(class_recalls)\n\n            if macro_precision + macro_recall > 0:\n                return round(2 * (macro_precision * macro_recall) / (macro_precision + macro_recall), 4)\n\n        return 0.0\n\n    def micro_avg_accuracy(self):\n        return self.accuracy(None)\n\n    def macro_avg_accuracy(self):\n        class_accuracy = [self.accuracy(class_name) for class_name in self.get_classes()]\n\n        if len(class_accuracy) > 0:\n            return round(sum(class_accuracy) / len(class_accuracy), 4)\n\n        return 0.0\n\n    def get_classes(self) -> List:\n        all_classes = set(itertools.chain(*[list(keys) for keys\n                                            in [self._tps.keys(), self._fps.keys(), self._tns.keys(),\n                                                self._fns.keys()]]))\n        all_classes = [class_name for class_name in all_classes if class_name is not None]\n        all_classes.sort()\n        return all_classes\n\n    def to_tsv(self):\n        return '{}\\t{}\\t{}\\t{}'.format(\n            self.precision(),\n            self.recall(),\n            self.accuracy(),\n            self.micro_avg_f_score(),\n        )\n\n    @staticmethod\n    def tsv_header(prefix=None):\n        if prefix:\n            return '{0}_PRECISION\\t{0}_RECALL\\t{0}_ACCURACY\\t{0}_F-SCORE'.format(\n                prefix)\n\n        return 'PRECISION\\tRECALL\\tACCURACY\\tF-SCORE'\n\n    @staticmethod\n    def to_empty_tsv():\n        return '\\t_\\t_\\t_\\t_'\n\n    def __str__(self):\n        all_classes = self.get_classes()\n        all_classes = [None] + all_classes\n        all_lines = [\n            '{0:<10}\\ttp: {1} - fp: {2} - fn: {3} - tn: {4} - precision: {5:.4f} - recall: {6:.4f} - accuracy: {7:.4f} - f1-score: {8:.4f}'.format(\n                self.name if class_name is None else class_name,\n                self.get_tp(class_name), self.get_fp(class_name), self.get_fn(class_name), self.get_tn(class_name),\n                self.precision(class_name), self.recall(class_name), self.accuracy(class_name),\n                self.f_score(class_name))\n            for class_name in all_classes]\n        return '\\n'.join(all_lines)\n\n\nclass EvaluationMetric(Enum):\n    MICRO_ACCURACY = 'micro-average accuracy'\n    MICRO_F1_SCORE = 'micro-average f1-score'\n    MACRO_ACCURACY = 'macro-average accuracy'\n    MACRO_F1_SCORE = 'macro-average f1-score'\n\n\nclass WeightExtractor(object):\n\n    def __init__(self, directory: Path, number_of_weights: int = 10):\n        self.weights_file = init_output_file(directory, 'weights.txt')\n        self.weights_dict = defaultdict(lambda: defaultdict(lambda: list()))\n        self.number_of_weights = number_of_weights\n\n    def extract_weights(self, state_dict, iteration):\n        for key in state_dict.keys():\n\n            vec = state_dict[key]\n            weights_to_watch = min(self.number_of_weights, reduce(lambda x, y: x * y, list(vec.size())))\n\n            if key not in self.weights_dict:\n                self._init_weights_index(key, state_dict, weights_to_watch)\n\n            for i in range(weights_to_watch):\n                vec = state_dict[key]\n                for index in self.weights_dict[key][i]:\n                    vec = vec[index]\n\n                value = vec.item()\n\n                with open(self.weights_file, 'a') as f:\n                    f.write('{}\\t{}\\t{}\\t{}\\n'.format(iteration, key, i, float(value)))\n\n    def _init_weights_index(self, key, state_dict, weights_to_watch):\n        indices = {}\n\n        i = 0\n        while len(indices) < weights_to_watch:\n            vec = state_dict[key]\n            cur_indices = []\n\n            for x in range(len(vec.size())):\n                index = random.randint(0, len(vec) - 1)\n                vec = vec[index]\n                cur_indices.append(index)\n\n            if cur_indices not in list(indices.values()):\n                indices[i] = cur_indices\n                i += 1\n\n        self.weights_dict[key] = indices\n\n\ndef clear_embeddings(sentences: List[Sentence], also_clear_word_embeddings=False):\n    \"\"\"\n    Clears the embeddings from all given sentences.\n    :param sentences: list of sentences\n    \"\"\"\n    for sentence in sentences:\n        sentence.clear_embeddings(also_clear_word_embeddings=also_clear_word_embeddings)\n\n\ndef init_output_file(base_path: Path, file_name: str) -> Path:\n    \"\"\"\n    Creates a local file.\n    :param base_path: the path to the directory\n    :param file_name: the file name\n    :return: the created file\n    \"\"\"\n    base_path.mkdir(parents=True, exist_ok=True)\n\n    file = base_path / file_name\n    open(file, \"w\", encoding='utf-8').close()\n    return file\n\n\ndef convert_labels_to_one_hot(label_list: List[List[str]], label_dict: Dictionary) -> List[List[int]]:\n    \"\"\"\n    Convert list of labels (strings) to a one hot list.\n    :param label_list: list of labels\n    :param label_dict: label dictionary\n    :return: converted label list\n    \"\"\"\n    return [[1 if l in labels else 0 for l in label_dict.get_items()] for labels in label_list]\n\n\ndef log_line(log):\n    log.info('-' * 100)\n\n\ndef add_file_handler(log, output_file):\n    init_output_file(output_file.parents[0], output_file.name)\n    fh = logging.FileHandler(output_file)\n    fh.setLevel(logging.INFO)\n    formatter = logging.Formatter('%(asctime)-15s %(message)s')\n    fh.setFormatter(formatter)\n    log.addHandler(fh)\n", "path": "flair/training_utils.py"}], "after_files": [{"content": "import itertools\nimport random\nimport logging\nfrom collections import defaultdict\nfrom enum import Enum\nfrom pathlib import Path\nfrom typing import List\nfrom flair.data import Dictionary, Sentence\nfrom functools import reduce\n\n\nclass Metric(object):\n\n    def __init__(self, name):\n        self.name = name\n\n        self._tps = defaultdict(int)\n        self._fps = defaultdict(int)\n        self._tns = defaultdict(int)\n        self._fns = defaultdict(int)\n\n    def add_tp(self, class_name):\n        self._tps[class_name] += 1\n\n    def add_tn(self, class_name):\n        self._tns[class_name] += 1\n\n    def add_fp(self, class_name):\n        self._fps[class_name] += 1\n\n    def add_fn(self, class_name):\n        self._fns[class_name] += 1\n\n    def get_tp(self, class_name=None):\n        if class_name is None:\n            return sum([self._tps[class_name] for class_name in self.get_classes()])\n        return self._tps[class_name]\n\n    def get_tn(self, class_name):\n        if class_name is None:\n            return sum([self._tns[class_name] for class_name in self.get_classes()])\n        return self._tns[class_name]\n\n    def get_fp(self, class_name):\n        if class_name is None:\n            return sum([self._fps[class_name] for class_name in self.get_classes()])\n        return self._fps[class_name]\n\n    def get_fn(self, class_name):\n        if class_name is None:\n            return sum([self._fns[class_name] for class_name in self.get_classes()])\n        return self._fns[class_name]\n\n    def precision(self, class_name=None):\n        if self.get_tp(class_name) + self.get_fp(class_name) > 0:\n            return round(self.get_tp(class_name) / (self.get_tp(class_name) + self.get_fp(class_name)), 4)\n        return 0.0\n\n    def recall(self, class_name=None):\n        if self.get_tp(class_name) + self.get_fn(class_name) > 0:\n            return round(self.get_tp(class_name) / (self.get_tp(class_name) + self.get_fn(class_name)), 4)\n        return 0.0\n\n    def f_score(self, class_name=None):\n        if self.precision(class_name) + self.recall(class_name) > 0:\n            return round(2 * (self.precision(class_name) * self.recall(class_name))\n                         / (self.precision(class_name) + self.recall(class_name)), 4)\n        return 0.0\n\n    def accuracy(self, class_name=None):\n        if self.get_tp(class_name) + self.get_tn(class_name) + self.get_fp(class_name) + self.get_fn(class_name) > 0:\n            return round(\n                (self.get_tp(class_name) + self.get_tn(class_name))\n                / (self.get_tp(class_name) + self.get_tn(class_name) + self.get_fp(class_name) + self.get_fn(class_name)),\n                4)\n        return 0.0\n\n    def micro_avg_f_score(self):\n        return self.f_score(None)\n\n    def macro_avg_f_score(self):\n        class_precisions = [self.precision(class_name) for class_name in self.get_classes()]\n        class_recalls = [self.recall(class_name) for class_name in self.get_classes()]\n\n        if len(class_precisions) > 0 and len(class_recalls) > 0:\n            macro_precision = sum(class_precisions) / len(class_precisions)\n            macro_recall = sum(class_recalls) / len(class_recalls)\n\n            if macro_precision + macro_recall > 0:\n                return round(2 * (macro_precision * macro_recall) / (macro_precision + macro_recall), 4)\n\n        return 0.0\n\n    def micro_avg_accuracy(self):\n        return self.accuracy(None)\n\n    def macro_avg_accuracy(self):\n        class_accuracy = [self.accuracy(class_name) for class_name in self.get_classes()]\n\n        if len(class_accuracy) > 0:\n            return round(sum(class_accuracy) / len(class_accuracy), 4)\n\n        return 0.0\n\n    def get_classes(self) -> List:\n        all_classes = set(itertools.chain(*[list(keys) for keys\n                                            in [self._tps.keys(), self._fps.keys(), self._tns.keys(),\n                                                self._fns.keys()]]))\n        all_classes = [class_name for class_name in all_classes if class_name is not None]\n        all_classes.sort()\n        return all_classes\n\n    def to_tsv(self):\n        return '{}\\t{}\\t{}\\t{}'.format(\n            self.precision(),\n            self.recall(),\n            self.accuracy(),\n            self.micro_avg_f_score(),\n        )\n\n    @staticmethod\n    def tsv_header(prefix=None):\n        if prefix:\n            return '{0}_PRECISION\\t{0}_RECALL\\t{0}_ACCURACY\\t{0}_F-SCORE'.format(\n                prefix)\n\n        return 'PRECISION\\tRECALL\\tACCURACY\\tF-SCORE'\n\n    @staticmethod\n    def to_empty_tsv():\n        return '\\t_\\t_\\t_\\t_'\n\n    def __str__(self):\n        all_classes = self.get_classes()\n        all_classes = [None] + all_classes\n        all_lines = [\n            '{0:<10}\\ttp: {1} - fp: {2} - fn: {3} - tn: {4} - precision: {5:.4f} - recall: {6:.4f} - accuracy: {7:.4f} - f1-score: {8:.4f}'.format(\n                self.name if class_name is None else class_name,\n                self.get_tp(class_name), self.get_fp(class_name), self.get_fn(class_name), self.get_tn(class_name),\n                self.precision(class_name), self.recall(class_name), self.accuracy(class_name),\n                self.f_score(class_name))\n            for class_name in all_classes]\n        return '\\n'.join(all_lines)\n\n\nclass EvaluationMetric(Enum):\n    MICRO_ACCURACY = 'micro-average accuracy'\n    MICRO_F1_SCORE = 'micro-average f1-score'\n    MACRO_ACCURACY = 'macro-average accuracy'\n    MACRO_F1_SCORE = 'macro-average f1-score'\n\n\nclass WeightExtractor(object):\n\n    def __init__(self, directory: Path, number_of_weights: int = 10):\n        self.weights_file = init_output_file(directory, 'weights.txt')\n        self.weights_dict = defaultdict(lambda: defaultdict(lambda: list()))\n        self.number_of_weights = number_of_weights\n\n    def extract_weights(self, state_dict, iteration):\n        for key in state_dict.keys():\n\n            vec = state_dict[key]\n            weights_to_watch = min(self.number_of_weights, reduce(lambda x, y: x * y, list(vec.size())))\n\n            if key not in self.weights_dict:\n                self._init_weights_index(key, state_dict, weights_to_watch)\n\n            for i in range(weights_to_watch):\n                vec = state_dict[key]\n                for index in self.weights_dict[key][i]:\n                    vec = vec[index]\n\n                value = vec.item()\n\n                with open(self.weights_file, 'a') as f:\n                    f.write('{}\\t{}\\t{}\\t{}\\n'.format(iteration, key, i, float(value)))\n\n    def _init_weights_index(self, key, state_dict, weights_to_watch):\n        indices = {}\n\n        i = 0\n        while len(indices) < weights_to_watch:\n            vec = state_dict[key]\n            cur_indices = []\n\n            for x in range(len(vec.size())):\n                index = random.randint(0, len(vec) - 1)\n                vec = vec[index]\n                cur_indices.append(index)\n\n            if cur_indices not in list(indices.values()):\n                indices[i] = cur_indices\n                i += 1\n\n        self.weights_dict[key] = indices\n\n\ndef clear_embeddings(sentences: List[Sentence], also_clear_word_embeddings=False):\n    \"\"\"\n    Clears the embeddings from all given sentences.\n    :param sentences: list of sentences\n    \"\"\"\n    for sentence in sentences:\n        sentence.clear_embeddings(also_clear_word_embeddings=also_clear_word_embeddings)\n\n\ndef init_output_file(base_path: Path, file_name: str) -> Path:\n    \"\"\"\n    Creates a local file.\n    :param base_path: the path to the directory\n    :param file_name: the file name\n    :return: the created file\n    \"\"\"\n    base_path.mkdir(parents=True, exist_ok=True)\n\n    file = base_path / file_name\n    open(file, \"w\", encoding='utf-8').close()\n    return file\n\n\ndef convert_labels_to_one_hot(label_list: List[List[str]], label_dict: Dictionary) -> List[List[int]]:\n    \"\"\"\n    Convert list of labels (strings) to a one hot list.\n    :param label_list: list of labels\n    :param label_dict: label dictionary\n    :return: converted label list\n    \"\"\"\n    return [[1 if l in labels else 0 for l in label_dict.get_items()] for labels in label_list]\n\n\ndef log_line(log):\n    log.info('-' * 100)\n\n\ndef add_file_handler(log, output_file):\n    init_output_file(output_file.parents[0], output_file.name)\n    fh = logging.FileHandler(output_file)\n    fh.setLevel(logging.INFO)\n    formatter = logging.Formatter('%(asctime)-15s %(message)s')\n    fh.setFormatter(formatter)\n    log.addHandler(fh)\n", "path": "flair/training_utils.py"}]}