We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
TextClassifier label predictions always have score 1.0
Hi guys,

I have trained my own TextClassifier following this tutorial: https://github.com/zalandoresearch/flair/blob/master/resources/docs/TUTORIAL_7_TRAINING_A_MODEL.md

I am using the option multi_label=False, as each sentence should be assigned only one label. In terms of embeddings, I use FlairEmbeddings mix-forward and mix-backward. 

The issue is that every time I predict the label of a new unseen sentence, I get a label score = 1.0. It seems that it never has any different value between 0.0 and 1.0. It is always 1.0. 

Is this the expected behavior? What am I doing wrong?

Thanks!

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `flair/models/text_classification_model.py`
Content:
```
1 import warnings
2 import logging
3 from pathlib import Path
4 from typing import List, Union
5 
6 import torch
7 import torch.nn as nn
8 
9 import flair.nn
10 import flair.embeddings
11 from flair.data import Dictionary, Sentence, Label
12 from flair.file_utils import cached_path
13 from flair.training_utils import convert_labels_to_one_hot, clear_embeddings
14 
15 
16 log = logging.getLogger("flair")
17 
18 
19 class TextClassifier(flair.nn.Model):
20     """
21     Text Classification Model
22     The model takes word embeddings, puts them into an RNN to obtain a text representation, and puts the
23     text representation in the end into a linear layer to get the actual class label.
24     The model can handle single and multi class data sets.
25     """
26 
27     def __init__(
28         self,
29         document_embeddings: flair.embeddings.DocumentEmbeddings,
30         label_dictionary: Dictionary,
31         multi_label: bool,
32     ):
33 
34         super(TextClassifier, self).__init__()
35 
36         self.document_embeddings: flair.embeddings.DocumentRNNEmbeddings = document_embeddings
37         self.label_dictionary: Dictionary = label_dictionary
38         self.multi_label = multi_label
39 
40         self.decoder = nn.Linear(
41             self.document_embeddings.embedding_length, len(self.label_dictionary)
42         )
43 
44         self._init_weights()
45 
46         if multi_label:
47             self.loss_function = nn.BCELoss()
48         else:
49             self.loss_function = nn.CrossEntropyLoss()
50 
51         # auto-spawn on GPU if available
52         self.to(flair.device)
53 
54     def _init_weights(self):
55         nn.init.xavier_uniform_(self.decoder.weight)
56 
57     def forward(self, sentences) -> List[List[float]]:
58         self.document_embeddings.embed(sentences)
59 
60         text_embedding_list = [
61             sentence.get_embedding().unsqueeze(0) for sentence in sentences
62         ]
63         text_embedding_tensor = torch.cat(text_embedding_list, 0).to(flair.device)
64 
65         label_scores = self.decoder(text_embedding_tensor)
66 
67         return label_scores
68 
69     def save(self, model_file: Union[str, Path]):
70         """
71         Saves the current model to the provided file.
72         :param model_file: the model file
73         """
74         model_state = {
75             "state_dict": self.state_dict(),
76             "document_embeddings": self.document_embeddings,
77             "label_dictionary": self.label_dictionary,
78             "multi_label": self.multi_label,
79         }
80         torch.save(model_state, str(model_file), pickle_protocol=4)
81 
82     def save_checkpoint(
83         self,
84         model_file: Union[str, Path],
85         optimizer_state: dict,
86         scheduler_state: dict,
87         epoch: int,
88         loss: float,
89     ):
90         """
91         Saves the current model to the provided file.
92         :param model_file: the model file
93         """
94         model_state = {
95             "state_dict": self.state_dict(),
96             "document_embeddings": self.document_embeddings,
97             "label_dictionary": self.label_dictionary,
98             "multi_label": self.multi_label,
99             "optimizer_state_dict": optimizer_state,
100             "scheduler_state_dict": scheduler_state,
101             "epoch": epoch,
102             "loss": loss,
103         }
104         torch.save(model_state, str(model_file), pickle_protocol=4)
105 
106     @classmethod
107     def load_from_file(cls, model_file: Union[str, Path]):
108         """
109         Loads the model from the given file.
110         :param model_file: the model file
111         :return: the loaded text classifier model
112         """
113         state = TextClassifier._load_state(model_file)
114 
115         model = TextClassifier(
116             document_embeddings=state["document_embeddings"],
117             label_dictionary=state["label_dictionary"],
118             multi_label=state["multi_label"],
119         )
120         model.load_state_dict(state["state_dict"])
121         model.eval()
122         model.to(flair.device)
123 
124         return model
125 
126     @classmethod
127     def load_checkpoint(cls, model_file: Union[str, Path]):
128         state = TextClassifier._load_state(model_file)
129         model = TextClassifier.load_from_file(model_file)
130 
131         epoch = state["epoch"] if "epoch" in state else None
132         loss = state["loss"] if "loss" in state else None
133         optimizer_state_dict = (
134             state["optimizer_state_dict"] if "optimizer_state_dict" in state else None
135         )
136         scheduler_state_dict = (
137             state["scheduler_state_dict"] if "scheduler_state_dict" in state else None
138         )
139 
140         return {
141             "model": model,
142             "epoch": epoch,
143             "loss": loss,
144             "optimizer_state_dict": optimizer_state_dict,
145             "scheduler_state_dict": scheduler_state_dict,
146         }
147 
148     @classmethod
149     def _load_state(cls, model_file: Union[str, Path]):
150         # ATTENTION: suppressing torch serialization warnings. This needs to be taken out once we sort out recursive
151         # serialization of torch objects
152         # https://docs.python.org/3/library/warnings.html#temporarily-suppressing-warnings
153         with warnings.catch_warnings():
154             warnings.filterwarnings("ignore")
155             # load_big_file is a workaround by https://github.com/highway11git to load models on some Mac/Windows setups
156             # see https://github.com/zalandoresearch/flair/issues/351
157             f = flair.file_utils.load_big_file(str(model_file))
158             state = torch.load(f, map_location=flair.device)
159             return state
160 
161     def forward_loss(self, sentences: Union[List[Sentence], Sentence]) -> torch.tensor:
162         scores = self.forward(sentences)
163         return self._calculate_loss(scores, sentences)
164 
165     def forward_labels_and_loss(
166         self, sentences: Union[Sentence, List[Sentence]]
167     ) -> (List[List[Label]], torch.tensor):
168         scores = self.forward(sentences)
169         labels = self._obtain_labels(scores)
170         loss = self._calculate_loss(scores, sentences)
171         return labels, loss
172 
173     def predict(
174         self, sentences: Union[Sentence, List[Sentence]], mini_batch_size: int = 32
175     ) -> List[Sentence]:
176         """
177         Predicts the class labels for the given sentences. The labels are directly added to the sentences.
178         :param sentences: list of sentences
179         :param mini_batch_size: mini batch size to use
180         :return: the list of sentences containing the labels
181         """
182         with torch.no_grad():
183             if type(sentences) is Sentence:
184                 sentences = [sentences]
185 
186             filtered_sentences = self._filter_empty_sentences(sentences)
187 
188             batches = [
189                 filtered_sentences[x : x + mini_batch_size]
190                 for x in range(0, len(filtered_sentences), mini_batch_size)
191             ]
192 
193             for batch in batches:
194                 scores = self.forward(batch)
195                 predicted_labels = self._obtain_labels(scores)
196 
197                 for (sentence, labels) in zip(batch, predicted_labels):
198                     sentence.labels = labels
199 
200                 clear_embeddings(batch)
201 
202             return sentences
203 
204     @staticmethod
205     def _filter_empty_sentences(sentences: List[Sentence]) -> List[Sentence]:
206         filtered_sentences = [sentence for sentence in sentences if sentence.tokens]
207         if len(sentences) != len(filtered_sentences):
208             log.warning(
209                 "Ignore {} sentence(s) with no tokens.".format(
210                     len(sentences) - len(filtered_sentences)
211                 )
212             )
213         return filtered_sentences
214 
215     def _calculate_loss(
216         self, scores: List[List[float]], sentences: List[Sentence]
217     ) -> float:
218         """
219         Calculates the loss.
220         :param scores: the prediction scores from the model
221         :param sentences: list of sentences
222         :return: loss value
223         """
224         if self.multi_label:
225             return self._calculate_multi_label_loss(scores, sentences)
226 
227         return self._calculate_single_label_loss(scores, sentences)
228 
229     def _obtain_labels(self, scores: List[List[float]]) -> List[List[Label]]:
230         """
231         Predicts the labels of sentences.
232         :param scores: the prediction scores from the model
233         :return: list of predicted labels
234         """
235 
236         if self.multi_label:
237             return [self._get_multi_label(s) for s in scores]
238 
239         return [self._get_single_label(s) for s in scores]
240 
241     def _get_multi_label(self, label_scores) -> List[Label]:
242         labels = []
243 
244         sigmoid = torch.nn.Sigmoid()
245 
246         results = list(map(lambda x: sigmoid(x), label_scores))
247         for idx, conf in enumerate(results):
248             if conf > 0.5:
249                 label = self.label_dictionary.get_item_for_index(idx)
250                 labels.append(Label(label, conf.item()))
251 
252         return labels
253 
254     def _get_single_label(self, label_scores) -> List[Label]:
255         conf, idx = torch.max(label_scores, 0)
256         label = self.label_dictionary.get_item_for_index(idx.item())
257 
258         return [Label(label, conf.item())]
259 
260     def _calculate_multi_label_loss(
261         self, label_scores, sentences: List[Sentence]
262     ) -> float:
263         sigmoid = nn.Sigmoid()
264         return self.loss_function(
265             sigmoid(label_scores), self._labels_to_one_hot(sentences)
266         )
267 
268     def _calculate_single_label_loss(
269         self, label_scores, sentences: List[Sentence]
270     ) -> float:
271         return self.loss_function(label_scores, self._labels_to_indices(sentences))
272 
273     def _labels_to_one_hot(self, sentences: List[Sentence]):
274         label_list = [sentence.get_label_names() for sentence in sentences]
275         one_hot = convert_labels_to_one_hot(label_list, self.label_dictionary)
276         one_hot = [torch.FloatTensor(l).unsqueeze(0) for l in one_hot]
277         one_hot = torch.cat(one_hot, 0).to(flair.device)
278         return one_hot
279 
280     def _labels_to_indices(self, sentences: List[Sentence]):
281         indices = [
282             torch.LongTensor(
283                 [
284                     self.label_dictionary.get_idx_for_item(label.value)
285                     for label in sentence.labels
286                 ]
287             )
288             for sentence in sentences
289         ]
290 
291         vec = torch.cat(indices, 0).to(flair.device)
292 
293         return vec
294 
295     @staticmethod
296     def load(model: str):
297         model_file = None
298         aws_resource_path = (
299             "https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/models-v0.4"
300         )
301         cache_dir = Path("models")
302 
303         if model.lower() == "de-offensive-language":
304             base_path = "/".join(
305                 [
306                     aws_resource_path,
307                     "TEXT-CLASSIFICATION_germ-eval-2018_task-1",
308                     "germ-eval-2018-task-1.pt",
309                 ]
310             )
311             model_file = cached_path(base_path, cache_dir=cache_dir)
312 
313         elif model.lower() == "en-sentiment":
314             base_path = "/".join(
315                 [aws_resource_path, "TEXT-CLASSIFICATION_imdb", "imdb.pt"]
316             )
317             model_file = cached_path(base_path, cache_dir=cache_dir)
318 
319         if model_file is not None:
320             return TextClassifier.load_from_file(model_file)
321
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py
--- a/flair/models/text_classification_model.py
+++ b/flair/models/text_classification_model.py
@@ -252,7 +252,8 @@
         return labels
 
     def _get_single_label(self, label_scores) -> List[Label]:
-        conf, idx = torch.max(label_scores, 0)
+        softmax = torch.nn.functional.softmax(label_scores, dim=0)
+        conf, idx = torch.max(softmax, 0)
         label = self.label_dictionary.get_item_for_index(idx.item())
 
         return [Label(label, conf.item())]

{"golden_diff": "diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py\n--- a/flair/models/text_classification_model.py\n+++ b/flair/models/text_classification_model.py\n@@ -252,7 +252,8 @@\n         return labels\n \n     def _get_single_label(self, label_scores) -> List[Label]:\n-        conf, idx = torch.max(label_scores, 0)\n+        softmax = torch.nn.functional.softmax(label_scores, dim=0)\n+        conf, idx = torch.max(softmax, 0)\n         label = self.label_dictionary.get_item_for_index(idx.item())\n \n         return [Label(label, conf.item())]\n", "issue": "TextClassifier label predictions always have score 1.0\nHi guys,\r\n\r\nI have trained my own TextClassifier following this tutorial: https://github.com/zalandoresearch/flair/blob/master/resources/docs/TUTORIAL_7_TRAINING_A_MODEL.md\r\n\r\nI am using the option multi_label=False, as each sentence should be assigned only one label. In terms of embeddings, I use FlairEmbeddings mix-forward and mix-backward. \r\n\r\nThe issue is that every time I predict the label of a new unseen sentence, I get a label score = 1.0. It seems that it never has any different value between 0.0 and 1.0. It is always 1.0. \r\n\r\nIs this the expected behavior? What am I doing wrong?\r\n\r\nThanks!\n", "before_files": [{"content": "import warnings\nimport logging\nfrom pathlib import Path\nfrom typing import List, Union\n\nimport torch\nimport torch.nn as nn\n\nimport flair.nn\nimport flair.embeddings\nfrom flair.data import Dictionary, Sentence, Label\nfrom flair.file_utils import cached_path\nfrom flair.training_utils import convert_labels_to_one_hot, clear_embeddings\n\n\nlog = logging.getLogger(\"flair\")\n\n\nclass TextClassifier(flair.nn.Model):\n    \"\"\"\n    Text Classification Model\n    The model takes word embeddings, puts them into an RNN to obtain a text representation, and puts the\n    text representation in the end into a linear layer to get the actual class label.\n    The model can handle single and multi class data sets.\n    \"\"\"\n\n    def __init__(\n        self,\n        document_embeddings: flair.embeddings.DocumentEmbeddings,\n        label_dictionary: Dictionary,\n        multi_label: bool,\n    ):\n\n        super(TextClassifier, self).__init__()\n\n        self.document_embeddings: flair.embeddings.DocumentRNNEmbeddings = document_embeddings\n        self.label_dictionary: Dictionary = label_dictionary\n        self.multi_label = multi_label\n\n        self.decoder = nn.Linear(\n            self.document_embeddings.embedding_length, len(self.label_dictionary)\n        )\n\n        self._init_weights()\n\n        if multi_label:\n            self.loss_function = nn.BCELoss()\n        else:\n            self.loss_function = nn.CrossEntropyLoss()\n\n        # auto-spawn on GPU if available\n        self.to(flair.device)\n\n    def _init_weights(self):\n        nn.init.xavier_uniform_(self.decoder.weight)\n\n    def forward(self, sentences) -> List[List[float]]:\n        self.document_embeddings.embed(sentences)\n\n        text_embedding_list = [\n            sentence.get_embedding().unsqueeze(0) for sentence in sentences\n        ]\n        text_embedding_tensor = torch.cat(text_embedding_list, 0).to(flair.device)\n\n        label_scores = self.decoder(text_embedding_tensor)\n\n        return label_scores\n\n    def save(self, model_file: Union[str, Path]):\n        \"\"\"\n        Saves the current model to the provided file.\n        :param model_file: the model file\n        \"\"\"\n        model_state = {\n            \"state_dict\": self.state_dict(),\n            \"document_embeddings\": self.document_embeddings,\n            \"label_dictionary\": self.label_dictionary,\n            \"multi_label\": self.multi_label,\n        }\n        torch.save(model_state, str(model_file), pickle_protocol=4)\n\n    def save_checkpoint(\n        self,\n        model_file: Union[str, Path],\n        optimizer_state: dict,\n        scheduler_state: dict,\n        epoch: int,\n        loss: float,\n    ):\n        \"\"\"\n        Saves the current model to the provided file.\n        :param model_file: the model file\n        \"\"\"\n        model_state = {\n            \"state_dict\": self.state_dict(),\n            \"document_embeddings\": self.document_embeddings,\n            \"label_dictionary\": self.label_dictionary,\n            \"multi_label\": self.multi_label,\n            \"optimizer_state_dict\": optimizer_state,\n            \"scheduler_state_dict\": scheduler_state,\n            \"epoch\": epoch,\n            \"loss\": loss,\n        }\n        torch.save(model_state, str(model_file), pickle_protocol=4)\n\n    @classmethod\n    def load_from_file(cls, model_file: Union[str, Path]):\n        \"\"\"\n        Loads the model from the given file.\n        :param model_file: the model file\n        :return: the loaded text classifier model\n        \"\"\"\n        state = TextClassifier._load_state(model_file)\n\n        model = TextClassifier(\n            document_embeddings=state[\"document_embeddings\"],\n            label_dictionary=state[\"label_dictionary\"],\n            multi_label=state[\"multi_label\"],\n        )\n        model.load_state_dict(state[\"state_dict\"])\n        model.eval()\n        model.to(flair.device)\n\n        return model\n\n    @classmethod\n    def load_checkpoint(cls, model_file: Union[str, Path]):\n        state = TextClassifier._load_state(model_file)\n        model = TextClassifier.load_from_file(model_file)\n\n        epoch = state[\"epoch\"] if \"epoch\" in state else None\n        loss = state[\"loss\"] if \"loss\" in state else None\n        optimizer_state_dict = (\n            state[\"optimizer_state_dict\"] if \"optimizer_state_dict\" in state else None\n        )\n        scheduler_state_dict = (\n            state[\"scheduler_state_dict\"] if \"scheduler_state_dict\" in state else None\n        )\n\n        return {\n            \"model\": model,\n            \"epoch\": epoch,\n            \"loss\": loss,\n            \"optimizer_state_dict\": optimizer_state_dict,\n            \"scheduler_state_dict\": scheduler_state_dict,\n        }\n\n    @classmethod\n    def _load_state(cls, model_file: Union[str, Path]):\n        # ATTENTION: suppressing torch serialization warnings. This needs to be taken out once we sort out recursive\n        # serialization of torch objects\n        # https://docs.python.org/3/library/warnings.html#temporarily-suppressing-warnings\n        with warnings.catch_warnings():\n            warnings.filterwarnings(\"ignore\")\n            # load_big_file is a workaround by https://github.com/highway11git to load models on some Mac/Windows setups\n            # see https://github.com/zalandoresearch/flair/issues/351\n            f = flair.file_utils.load_big_file(str(model_file))\n            state = torch.load(f, map_location=flair.device)\n            return state\n\n    def forward_loss(self, sentences: Union[List[Sentence], Sentence]) -> torch.tensor:\n        scores = self.forward(sentences)\n        return self._calculate_loss(scores, sentences)\n\n    def forward_labels_and_loss(\n        self, sentences: Union[Sentence, List[Sentence]]\n    ) -> (List[List[Label]], torch.tensor):\n        scores = self.forward(sentences)\n        labels = self._obtain_labels(scores)\n        loss = self._calculate_loss(scores, sentences)\n        return labels, loss\n\n    def predict(\n        self, sentences: Union[Sentence, List[Sentence]], mini_batch_size: int = 32\n    ) -> List[Sentence]:\n        \"\"\"\n        Predicts the class labels for the given sentences. The labels are directly added to the sentences.\n        :param sentences: list of sentences\n        :param mini_batch_size: mini batch size to use\n        :return: the list of sentences containing the labels\n        \"\"\"\n        with torch.no_grad():\n            if type(sentences) is Sentence:\n                sentences = [sentences]\n\n            filtered_sentences = self._filter_empty_sentences(sentences)\n\n            batches = [\n                filtered_sentences[x : x + mini_batch_size]\n                for x in range(0, len(filtered_sentences), mini_batch_size)\n            ]\n\n            for batch in batches:\n                scores = self.forward(batch)\n                predicted_labels = self._obtain_labels(scores)\n\n                for (sentence, labels) in zip(batch, predicted_labels):\n                    sentence.labels = labels\n\n                clear_embeddings(batch)\n\n            return sentences\n\n    @staticmethod\n    def _filter_empty_sentences(sentences: List[Sentence]) -> List[Sentence]:\n        filtered_sentences = [sentence for sentence in sentences if sentence.tokens]\n        if len(sentences) != len(filtered_sentences):\n            log.warning(\n                \"Ignore {} sentence(s) with no tokens.\".format(\n                    len(sentences) - len(filtered_sentences)\n                )\n            )\n        return filtered_sentences\n\n    def _calculate_loss(\n        self, scores: List[List[float]], sentences: List[Sentence]\n    ) -> float:\n        \"\"\"\n        Calculates the loss.\n        :param scores: the prediction scores from the model\n        :param sentences: list of sentences\n        :return: loss value\n        \"\"\"\n        if self.multi_label:\n            return self._calculate_multi_label_loss(scores, sentences)\n\n        return self._calculate_single_label_loss(scores, sentences)\n\n    def _obtain_labels(self, scores: List[List[float]]) -> List[List[Label]]:\n        \"\"\"\n        Predicts the labels of sentences.\n        :param scores: the prediction scores from the model\n        :return: list of predicted labels\n        \"\"\"\n\n        if self.multi_label:\n            return [self._get_multi_label(s) for s in scores]\n\n        return [self._get_single_label(s) for s in scores]\n\n    def _get_multi_label(self, label_scores) -> List[Label]:\n        labels = []\n\n        sigmoid = torch.nn.Sigmoid()\n\n        results = list(map(lambda x: sigmoid(x), label_scores))\n        for idx, conf in enumerate(results):\n            if conf > 0.5:\n                label = self.label_dictionary.get_item_for_index(idx)\n                labels.append(Label(label, conf.item()))\n\n        return labels\n\n    def _get_single_label(self, label_scores) -> List[Label]:\n        conf, idx = torch.max(label_scores, 0)\n        label = self.label_dictionary.get_item_for_index(idx.item())\n\n        return [Label(label, conf.item())]\n\n    def _calculate_multi_label_loss(\n        self, label_scores, sentences: List[Sentence]\n    ) -> float:\n        sigmoid = nn.Sigmoid()\n        return self.loss_function(\n            sigmoid(label_scores), self._labels_to_one_hot(sentences)\n        )\n\n    def _calculate_single_label_loss(\n        self, label_scores, sentences: List[Sentence]\n    ) -> float:\n        return self.loss_function(label_scores, self._labels_to_indices(sentences))\n\n    def _labels_to_one_hot(self, sentences: List[Sentence]):\n        label_list = [sentence.get_label_names() for sentence in sentences]\n        one_hot = convert_labels_to_one_hot(label_list, self.label_dictionary)\n        one_hot = [torch.FloatTensor(l).unsqueeze(0) for l in one_hot]\n        one_hot = torch.cat(one_hot, 0).to(flair.device)\n        return one_hot\n\n    def _labels_to_indices(self, sentences: List[Sentence]):\n        indices = [\n            torch.LongTensor(\n                [\n                    self.label_dictionary.get_idx_for_item(label.value)\n                    for label in sentence.labels\n                ]\n            )\n            for sentence in sentences\n        ]\n\n        vec = torch.cat(indices, 0).to(flair.device)\n\n        return vec\n\n    @staticmethod\n    def load(model: str):\n        model_file = None\n        aws_resource_path = (\n            \"https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/models-v0.4\"\n        )\n        cache_dir = Path(\"models\")\n\n        if model.lower() == \"de-offensive-language\":\n            base_path = \"/\".join(\n                [\n                    aws_resource_path,\n                    \"TEXT-CLASSIFICATION_germ-eval-2018_task-1\",\n                    \"germ-eval-2018-task-1.pt\",\n                ]\n            )\n            model_file = cached_path(base_path, cache_dir=cache_dir)\n\n        elif model.lower() == \"en-sentiment\":\n            base_path = \"/\".join(\n                [aws_resource_path, \"TEXT-CLASSIFICATION_imdb\", \"imdb.pt\"]\n            )\n            model_file = cached_path(base_path, cache_dir=cache_dir)\n\n        if model_file is not None:\n            return TextClassifier.load_from_file(model_file)\n", "path": "flair/models/text_classification_model.py"}], "after_files": [{"content": "import warnings\nimport logging\nfrom pathlib import Path\nfrom typing import List, Union\n\nimport torch\nimport torch.nn as nn\n\nimport flair.nn\nimport flair.embeddings\nfrom flair.data import Dictionary, Sentence, Label\nfrom flair.file_utils import cached_path\nfrom flair.training_utils import convert_labels_to_one_hot, clear_embeddings\n\n\nlog = logging.getLogger(\"flair\")\n\n\nclass TextClassifier(flair.nn.Model):\n    \"\"\"\n    Text Classification Model\n    The model takes word embeddings, puts them into an RNN to obtain a text representation, and puts the\n    text representation in the end into a linear layer to get the actual class label.\n    The model can handle single and multi class data sets.\n    \"\"\"\n\n    def __init__(\n        self,\n        document_embeddings: flair.embeddings.DocumentEmbeddings,\n        label_dictionary: Dictionary,\n        multi_label: bool,\n    ):\n\n        super(TextClassifier, self).__init__()\n\n        self.document_embeddings: flair.embeddings.DocumentRNNEmbeddings = document_embeddings\n        self.label_dictionary: Dictionary = label_dictionary\n        self.multi_label = multi_label\n\n        self.decoder = nn.Linear(\n            self.document_embeddings.embedding_length, len(self.label_dictionary)\n        )\n\n        self._init_weights()\n\n        if multi_label:\n            self.loss_function = nn.BCELoss()\n        else:\n            self.loss_function = nn.CrossEntropyLoss()\n\n        # auto-spawn on GPU if available\n        self.to(flair.device)\n\n    def _init_weights(self):\n        nn.init.xavier_uniform_(self.decoder.weight)\n\n    def forward(self, sentences) -> List[List[float]]:\n        self.document_embeddings.embed(sentences)\n\n        text_embedding_list = [\n            sentence.get_embedding().unsqueeze(0) for sentence in sentences\n        ]\n        text_embedding_tensor = torch.cat(text_embedding_list, 0).to(flair.device)\n\n        label_scores = self.decoder(text_embedding_tensor)\n\n        return label_scores\n\n    def save(self, model_file: Union[str, Path]):\n        \"\"\"\n        Saves the current model to the provided file.\n        :param model_file: the model file\n        \"\"\"\n        model_state = {\n            \"state_dict\": self.state_dict(),\n            \"document_embeddings\": self.document_embeddings,\n            \"label_dictionary\": self.label_dictionary,\n            \"multi_label\": self.multi_label,\n        }\n        torch.save(model_state, str(model_file), pickle_protocol=4)\n\n    def save_checkpoint(\n        self,\n        model_file: Union[str, Path],\n        optimizer_state: dict,\n        scheduler_state: dict,\n        epoch: int,\n        loss: float,\n    ):\n        \"\"\"\n        Saves the current model to the provided file.\n        :param model_file: the model file\n        \"\"\"\n        model_state = {\n            \"state_dict\": self.state_dict(),\n            \"document_embeddings\": self.document_embeddings,\n            \"label_dictionary\": self.label_dictionary,\n            \"multi_label\": self.multi_label,\n            \"optimizer_state_dict\": optimizer_state,\n            \"scheduler_state_dict\": scheduler_state,\n            \"epoch\": epoch,\n            \"loss\": loss,\n        }\n        torch.save(model_state, str(model_file), pickle_protocol=4)\n\n    @classmethod\n    def load_from_file(cls, model_file: Union[str, Path]):\n        \"\"\"\n        Loads the model from the given file.\n        :param model_file: the model file\n        :return: the loaded text classifier model\n        \"\"\"\n        state = TextClassifier._load_state(model_file)\n\n        model = TextClassifier(\n            document_embeddings=state[\"document_embeddings\"],\n            label_dictionary=state[\"label_dictionary\"],\n            multi_label=state[\"multi_label\"],\n        )\n        model.load_state_dict(state[\"state_dict\"])\n        model.eval()\n        model.to(flair.device)\n\n        return model\n\n    @classmethod\n    def load_checkpoint(cls, model_file: Union[str, Path]):\n        state = TextClassifier._load_state(model_file)\n        model = TextClassifier.load_from_file(model_file)\n\n        epoch = state[\"epoch\"] if \"epoch\" in state else None\n        loss = state[\"loss\"] if \"loss\" in state else None\n        optimizer_state_dict = (\n            state[\"optimizer_state_dict\"] if \"optimizer_state_dict\" in state else None\n        )\n        scheduler_state_dict = (\n            state[\"scheduler_state_dict\"] if \"scheduler_state_dict\" in state else None\n        )\n\n        return {\n            \"model\": model,\n            \"epoch\": epoch,\n            \"loss\": loss,\n            \"optimizer_state_dict\": optimizer_state_dict,\n            \"scheduler_state_dict\": scheduler_state_dict,\n        }\n\n    @classmethod\n    def _load_state(cls, model_file: Union[str, Path]):\n        # ATTENTION: suppressing torch serialization warnings. This needs to be taken out once we sort out recursive\n        # serialization of torch objects\n        # https://docs.python.org/3/library/warnings.html#temporarily-suppressing-warnings\n        with warnings.catch_warnings():\n            warnings.filterwarnings(\"ignore\")\n            # load_big_file is a workaround by https://github.com/highway11git to load models on some Mac/Windows setups\n            # see https://github.com/zalandoresearch/flair/issues/351\n            f = flair.file_utils.load_big_file(str(model_file))\n            state = torch.load(f, map_location=flair.device)\n            return state\n\n    def forward_loss(self, sentences: Union[List[Sentence], Sentence]) -> torch.tensor:\n        scores = self.forward(sentences)\n        return self._calculate_loss(scores, sentences)\n\n    def forward_labels_and_loss(\n        self, sentences: Union[Sentence, List[Sentence]]\n    ) -> (List[List[Label]], torch.tensor):\n        scores = self.forward(sentences)\n        labels = self._obtain_labels(scores)\n        loss = self._calculate_loss(scores, sentences)\n        return labels, loss\n\n    def predict(\n        self, sentences: Union[Sentence, List[Sentence]], mini_batch_size: int = 32\n    ) -> List[Sentence]:\n        \"\"\"\n        Predicts the class labels for the given sentences. The labels are directly added to the sentences.\n        :param sentences: list of sentences\n        :param mini_batch_size: mini batch size to use\n        :return: the list of sentences containing the labels\n        \"\"\"\n        with torch.no_grad():\n            if type(sentences) is Sentence:\n                sentences = [sentences]\n\n            filtered_sentences = self._filter_empty_sentences(sentences)\n\n            batches = [\n                filtered_sentences[x : x + mini_batch_size]\n                for x in range(0, len(filtered_sentences), mini_batch_size)\n            ]\n\n            for batch in batches:\n                scores = self.forward(batch)\n                predicted_labels = self._obtain_labels(scores)\n\n                for (sentence, labels) in zip(batch, predicted_labels):\n                    sentence.labels = labels\n\n                clear_embeddings(batch)\n\n            return sentences\n\n    @staticmethod\n    def _filter_empty_sentences(sentences: List[Sentence]) -> List[Sentence]:\n        filtered_sentences = [sentence for sentence in sentences if sentence.tokens]\n        if len(sentences) != len(filtered_sentences):\n            log.warning(\n                \"Ignore {} sentence(s) with no tokens.\".format(\n                    len(sentences) - len(filtered_sentences)\n                )\n            )\n        return filtered_sentences\n\n    def _calculate_loss(\n        self, scores: List[List[float]], sentences: List[Sentence]\n    ) -> float:\n        \"\"\"\n        Calculates the loss.\n        :param scores: the prediction scores from the model\n        :param sentences: list of sentences\n        :return: loss value\n        \"\"\"\n        if self.multi_label:\n            return self._calculate_multi_label_loss(scores, sentences)\n\n        return self._calculate_single_label_loss(scores, sentences)\n\n    def _obtain_labels(self, scores: List[List[float]]) -> List[List[Label]]:\n        \"\"\"\n        Predicts the labels of sentences.\n        :param scores: the prediction scores from the model\n        :return: list of predicted labels\n        \"\"\"\n\n        if self.multi_label:\n            return [self._get_multi_label(s) for s in scores]\n\n        return [self._get_single_label(s) for s in scores]\n\n    def _get_multi_label(self, label_scores) -> List[Label]:\n        labels = []\n\n        sigmoid = torch.nn.Sigmoid()\n\n        results = list(map(lambda x: sigmoid(x), label_scores))\n        for idx, conf in enumerate(results):\n            if conf > 0.5:\n                label = self.label_dictionary.get_item_for_index(idx)\n                labels.append(Label(label, conf.item()))\n\n        return labels\n\n    def _get_single_label(self, label_scores) -> List[Label]:\n        softmax = torch.nn.functional.softmax(label_scores, dim=0)\n        conf, idx = torch.max(softmax, 0)\n        label = self.label_dictionary.get_item_for_index(idx.item())\n\n        return [Label(label, conf.item())]\n\n    def _calculate_multi_label_loss(\n        self, label_scores, sentences: List[Sentence]\n    ) -> float:\n        sigmoid = nn.Sigmoid()\n        return self.loss_function(\n            sigmoid(label_scores), self._labels_to_one_hot(sentences)\n        )\n\n    def _calculate_single_label_loss(\n        self, label_scores, sentences: List[Sentence]\n    ) -> float:\n        return self.loss_function(label_scores, self._labels_to_indices(sentences))\n\n    def _labels_to_one_hot(self, sentences: List[Sentence]):\n        label_list = [sentence.get_label_names() for sentence in sentences]\n        one_hot = convert_labels_to_one_hot(label_list, self.label_dictionary)\n        one_hot = [torch.FloatTensor(l).unsqueeze(0) for l in one_hot]\n        one_hot = torch.cat(one_hot, 0).to(flair.device)\n        return one_hot\n\n    def _labels_to_indices(self, sentences: List[Sentence]):\n        indices = [\n            torch.LongTensor(\n                [\n                    self.label_dictionary.get_idx_for_item(label.value)\n                    for label in sentence.labels\n                ]\n            )\n            for sentence in sentences\n        ]\n\n        vec = torch.cat(indices, 0).to(flair.device)\n\n        return vec\n\n    @staticmethod\n    def load(model: str):\n        model_file = None\n        aws_resource_path = (\n            \"https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/models-v0.4\"\n        )\n        cache_dir = Path(\"models\")\n\n        if model.lower() == \"de-offensive-language\":\n            base_path = \"/\".join(\n                [\n                    aws_resource_path,\n                    \"TEXT-CLASSIFICATION_germ-eval-2018_task-1\",\n                    \"germ-eval-2018-task-1.pt\",\n                ]\n            )\n            model_file = cached_path(base_path, cache_dir=cache_dir)\n\n        elif model.lower() == \"en-sentiment\":\n            base_path = \"/\".join(\n                [aws_resource_path, \"TEXT-CLASSIFICATION_imdb\", \"imdb.pt\"]\n            )\n            model_file = cached_path(base_path, cache_dir=cache_dir)\n\n        if model_file is not None:\n            return TextClassifier.load_from_file(model_file)\n", "path": "flair/models/text_classification_model.py"}]}