We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
When running the calculate perplexity, got the error
Here is the error,  sounds likely it needs GPU, would you suggestion what I should I modify in the code of perplexity calculation?
File "/flair/models/language_model.py", line 69, in forward
    encoded = self.encoder(input)
  File ".local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)
  File ".local/lib/python3.6/site-packages/torch/nn/modules/sparse.py", line 110, in forward
    self.norm_type, self.scale_grad_by_freq, self.sparse)
  File ".local/lib/python3.6/site-packages/torch/nn/functional.py", line 1110, in embedding
    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
RuntimeError: Expected object of type torch.cuda.LongTensor but found type torch.LongTensor for argument #3 'index'

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `flair/models/language_model.py`
Content:
```
1 from pathlib import Path
2 
3 import torch.nn as nn
4 import torch
5 import math
6 from typing import Union, Tuple
7 from typing import List
8 
9 from torch.optim import Optimizer
10 
11 import flair
12 from flair.data import Dictionary
13 
14 
15 class LanguageModel(nn.Module):
16     """Container module with an encoder, a recurrent module, and a decoder."""
17 
18     def __init__(self,
19                  dictionary: Dictionary,
20                  is_forward_lm: bool,
21                  hidden_size: int,
22                  nlayers: int,
23                  embedding_size: int = 100,
24                  nout=None,
25                  dropout=0.1):
26 
27         super(LanguageModel, self).__init__()
28 
29         self.dictionary = dictionary
30         self.is_forward_lm: bool = is_forward_lm
31 
32         self.dropout = dropout
33         self.hidden_size = hidden_size
34         self.embedding_size = embedding_size
35         self.nlayers = nlayers
36 
37         self.drop = nn.Dropout(dropout)
38         self.encoder = nn.Embedding(len(dictionary), embedding_size)
39 
40         if nlayers == 1:
41             self.rnn = nn.LSTM(embedding_size, hidden_size, nlayers)
42         else:
43             self.rnn = nn.LSTM(embedding_size, hidden_size, nlayers, dropout=dropout)
44 
45         self.hidden = None
46 
47         self.nout = nout
48         if nout is not None:
49             self.proj = nn.Linear(hidden_size, nout)
50             self.initialize(self.proj.weight)
51             self.decoder = nn.Linear(nout, len(dictionary))
52         else:
53             self.proj = None
54             self.decoder = nn.Linear(hidden_size, len(dictionary))
55 
56         self.init_weights()
57 
58         # auto-spawn on GPU if available
59         self.to(flair.device)
60 
61     def init_weights(self):
62         initrange = 0.1
63         self.encoder.weight.detach().uniform_(-initrange, initrange)
64         self.decoder.bias.detach().fill_(0)
65         self.decoder.weight.detach().uniform_(-initrange, initrange)
66 
67     def set_hidden(self, hidden):
68         self.hidden = hidden
69 
70     def forward(self, input, hidden, ordered_sequence_lengths=None):
71         encoded = self.encoder(input)
72         emb = self.drop(encoded)
73 
74         self.rnn.flatten_parameters()
75 
76         output, hidden = self.rnn(emb, hidden)
77 
78         if self.proj is not None:
79             output = self.proj(output)
80 
81         output = self.drop(output)
82 
83         decoded = self.decoder(output.view(output.size(0) * output.size(1), output.size(2)))
84 
85         return decoded.view(output.size(0), output.size(1), decoded.size(1)), output, hidden
86 
87     def init_hidden(self, bsz):
88         weight = next(self.parameters()).detach()
89         return (weight.new(self.nlayers, bsz, self.hidden_size).zero_().clone().detach(),
90                 weight.new(self.nlayers, bsz, self.hidden_size).zero_().clone().detach())
91 
92     def get_representation(self, strings: List[str], chars_per_chunk: int = 512):
93 
94         # cut up the input into chunks of max charlength = chunk_size
95         longest = len(strings[0])
96         chunks = []
97         splice_begin = 0
98         for splice_end in range(chars_per_chunk, longest, chars_per_chunk):
99             chunks.append([text[splice_begin:splice_end] for text in strings])
100             splice_begin = splice_end
101 
102         chunks.append([text[splice_begin:longest] for text in strings])
103         hidden = self.init_hidden(len(chunks[0]))
104 
105         output_parts = []
106 
107         # push each chunk through the RNN language model
108         for chunk in chunks:
109 
110             sequences_as_char_indices: List[List[int]] = []
111             for string in chunk:
112                 char_indices = [self.dictionary.get_idx_for_item(char) for char in string]
113                 sequences_as_char_indices.append(char_indices)
114 
115             batch = torch.LongTensor(sequences_as_char_indices).transpose(0, 1)
116             batch = batch.to(flair.device)
117 
118             prediction, rnn_output, hidden = self.forward(batch, hidden)
119             rnn_output = rnn_output.detach()
120 
121             output_parts.append(rnn_output)
122 
123         # concatenate all chunks to make final output
124         output = torch.cat(output_parts)
125 
126         return output
127 
128     def get_output(self, text: str):
129         char_indices = [self.dictionary.get_idx_for_item(char) for char in text]
130         input_vector = torch.LongTensor([char_indices]).transpose(0, 1)
131 
132         hidden = self.init_hidden(1)
133         prediction, rnn_output, hidden = self.forward(input_vector, hidden)
134 
135         return self.repackage_hidden(hidden)
136 
137     def repackage_hidden(self, h):
138         """Wraps hidden states in new Variables, to detach them from their history."""
139         if type(h) == torch.Tensor:
140             return h.clone().detach()
141         else:
142             return tuple(self.repackage_hidden(v) for v in h)
143 
144     def initialize(self, matrix):
145         in_, out_ = matrix.size()
146         stdv = math.sqrt(3. / (in_ + out_))
147         matrix.detach().uniform_(-stdv, stdv)
148 
149     @classmethod
150     def load_language_model(cls, model_file: Union[Path, str]):
151 
152         state = torch.load(str(model_file), map_location=flair.device)
153 
154         model = LanguageModel(state['dictionary'],
155                               state['is_forward_lm'],
156                               state['hidden_size'],
157                               state['nlayers'],
158                               state['embedding_size'],
159                               state['nout'],
160                               state['dropout'])
161         model.load_state_dict(state['state_dict'])
162         model.eval()
163         model.to(flair.device)
164 
165         return model
166 
167     @classmethod
168     def load_checkpoint(cls, model_file: Path):
169         state = torch.load(str(model_file), map_location=flair.device)
170 
171         epoch = state['epoch'] if 'epoch' in state else None
172         split = state['split'] if 'split' in state else None
173         loss = state['loss'] if 'loss' in state else None
174         optimizer_state_dict = state['optimizer_state_dict'] if 'optimizer_state_dict' in state else None
175 
176         model = LanguageModel(state['dictionary'],
177                               state['is_forward_lm'],
178                               state['hidden_size'],
179                               state['nlayers'],
180                               state['embedding_size'],
181                               state['nout'],
182                               state['dropout'])
183         model.load_state_dict(state['state_dict'])
184         model.eval()
185         model.to(flair.device)
186 
187         return {'model': model, 'epoch': epoch, 'split': split, 'loss': loss,
188                 'optimizer_state_dict': optimizer_state_dict}
189 
190     def save_checkpoint(self, file: Path, optimizer: Optimizer, epoch: int, split: int, loss: float):
191         model_state = {
192             'state_dict': self.state_dict(),
193             'dictionary': self.dictionary,
194             'is_forward_lm': self.is_forward_lm,
195             'hidden_size': self.hidden_size,
196             'nlayers': self.nlayers,
197             'embedding_size': self.embedding_size,
198             'nout': self.nout,
199             'dropout': self.dropout,
200             'optimizer_state_dict': optimizer.state_dict(),
201             'epoch': epoch,
202             'split': split,
203             'loss': loss
204         }
205 
206         torch.save(model_state, str(file), pickle_protocol=4)
207 
208     def save(self, file: Path):
209         model_state = {
210             'state_dict': self.state_dict(),
211             'dictionary': self.dictionary,
212             'is_forward_lm': self.is_forward_lm,
213             'hidden_size': self.hidden_size,
214             'nlayers': self.nlayers,
215             'embedding_size': self.embedding_size,
216             'nout': self.nout,
217             'dropout': self.dropout
218         }
219 
220         torch.save(model_state, str(file), pickle_protocol=4)
221 
222     def generate_text(self, prefix: str = '\n', number_of_characters: int = 1000, temperature: float = 1.0,
223                       break_on_suffix=None) -> Tuple[str, float]:
224 
225         if prefix == '':
226             prefix = '\n'
227 
228         with torch.no_grad():
229             characters = []
230 
231             idx2item = self.dictionary.idx2item
232 
233             # initial hidden state
234             hidden = self.init_hidden(1)
235 
236             if len(prefix) > 1:
237 
238                 char_tensors = []
239                 for character in prefix[:-1]:
240                     char_tensors.append(
241                         torch.tensor(self.dictionary.get_idx_for_item(character)).unsqueeze(0).unsqueeze(0))
242 
243                 input = torch.cat(char_tensors)
244                 if torch.cuda.is_available():
245                     input = input.cuda()
246 
247                 prediction, _, hidden = self.forward(input, hidden)
248 
249             input = torch.tensor(self.dictionary.get_idx_for_item(prefix[-1])).unsqueeze(0).unsqueeze(0)
250 
251             log_prob = 0.
252 
253             for i in range(number_of_characters):
254 
255                 if torch.cuda.is_available():
256                     input = input.cuda()
257 
258                 # get predicted weights
259                 prediction, _, hidden = self.forward(input, hidden)
260                 prediction = prediction.squeeze().detach()
261                 decoder_output = prediction
262 
263                 # divide by temperature
264                 prediction = prediction.div(temperature)
265 
266                 # to prevent overflow problem with small temperature values, substract largest value from all
267                 # this makes a vector in which the largest value is 0
268                 max = torch.max(prediction)
269                 prediction -= max
270 
271                 # compute word weights with exponential function
272                 word_weights = prediction.exp().cpu()
273 
274                 # try sampling multinomial distribution for next character
275                 try:
276                     word_idx = torch.multinomial(word_weights, 1)[0]
277                 except:
278                     word_idx = torch.tensor(0)
279 
280                 # print(word_idx)
281                 prob = decoder_output[word_idx]
282                 log_prob += prob
283 
284                 input = word_idx.detach().unsqueeze(0).unsqueeze(0)
285                 word = idx2item[word_idx].decode('UTF-8')
286                 characters.append(word)
287 
288                 if break_on_suffix is not None:
289                     if ''.join(characters).endswith(break_on_suffix):
290                         break
291 
292             text = prefix + ''.join(characters)
293 
294             log_prob = log_prob.item()
295             log_prob /= len(characters)
296 
297             if not self.is_forward_lm:
298                 text = text[::-1]
299 
300             return text, log_prob
301 
302     def calculate_perplexity(self, text: str) -> float:
303 
304         if not self.is_forward_lm:
305             text = text[::-1]
306 
307         # input ids
308         input = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[:-1]]).unsqueeze(1)
309 
310         # push list of character IDs through model
311         hidden = self.init_hidden(1)
312         prediction, _, hidden = self.forward(input, hidden)
313 
314         # the target is always the next character
315         targets = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[1:]])
316 
317         # use cross entropy loss to compare output of forward pass with targets
318         cross_entroy_loss = torch.nn.CrossEntropyLoss()
319         loss = cross_entroy_loss(prediction.view(-1, len(self.dictionary)), targets).item()
320 
321         # exponentiate cross-entropy loss to calculate perplexity
322         perplexity = math.exp(loss)
323 
324         return perplexity
325
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/flair/models/language_model.py b/flair/models/language_model.py
--- a/flair/models/language_model.py
+++ b/flair/models/language_model.py
@@ -306,6 +306,7 @@
 
         # input ids
         input = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[:-1]]).unsqueeze(1)
+        input = input.to(flair.device)
 
         # push list of character IDs through model
         hidden = self.init_hidden(1)
@@ -313,6 +314,7 @@
 
         # the target is always the next character
         targets = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[1:]])
+        targets = targets.to(flair.device)
 
         # use cross entropy loss to compare output of forward pass with targets
         cross_entroy_loss = torch.nn.CrossEntropyLoss()

{"golden_diff": "diff --git a/flair/models/language_model.py b/flair/models/language_model.py\n--- a/flair/models/language_model.py\n+++ b/flair/models/language_model.py\n@@ -306,6 +306,7 @@\n \n         # input ids\n         input = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[:-1]]).unsqueeze(1)\n+        input = input.to(flair.device)\n \n         # push list of character IDs through model\n         hidden = self.init_hidden(1)\n@@ -313,6 +314,7 @@\n \n         # the target is always the next character\n         targets = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[1:]])\n+        targets = targets.to(flair.device)\n \n         # use cross entropy loss to compare output of forward pass with targets\n         cross_entroy_loss = torch.nn.CrossEntropyLoss()\n", "issue": "When running the calculate perplexity, got the error\nHere is the error,  sounds likely it needs GPU, would you suggestion what I should I modify in the code of perplexity calculation?\r\nFile \"/flair/models/language_model.py\", line 69, in forward\r\n    encoded = self.encoder(input)\r\n  File \".local/lib/python3.6/site-packages/torch/nn/modules/module.py\", line 477, in __call__\r\n    result = self.forward(*input, **kwargs)\r\n  File \".local/lib/python3.6/site-packages/torch/nn/modules/sparse.py\", line 110, in forward\r\n    self.norm_type, self.scale_grad_by_freq, self.sparse)\r\n  File \".local/lib/python3.6/site-packages/torch/nn/functional.py\", line 1110, in embedding\r\n    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)\r\nRuntimeError: Expected object of type torch.cuda.LongTensor but found type torch.LongTensor for argument #3 'index'\n", "before_files": [{"content": "from pathlib import Path\n\nimport torch.nn as nn\nimport torch\nimport math\nfrom typing import Union, Tuple\nfrom typing import List\n\nfrom torch.optim import Optimizer\n\nimport flair\nfrom flair.data import Dictionary\n\n\nclass LanguageModel(nn.Module):\n    \"\"\"Container module with an encoder, a recurrent module, and a decoder.\"\"\"\n\n    def __init__(self,\n                 dictionary: Dictionary,\n                 is_forward_lm: bool,\n                 hidden_size: int,\n                 nlayers: int,\n                 embedding_size: int = 100,\n                 nout=None,\n                 dropout=0.1):\n\n        super(LanguageModel, self).__init__()\n\n        self.dictionary = dictionary\n        self.is_forward_lm: bool = is_forward_lm\n\n        self.dropout = dropout\n        self.hidden_size = hidden_size\n        self.embedding_size = embedding_size\n        self.nlayers = nlayers\n\n        self.drop = nn.Dropout(dropout)\n        self.encoder = nn.Embedding(len(dictionary), embedding_size)\n\n        if nlayers == 1:\n            self.rnn = nn.LSTM(embedding_size, hidden_size, nlayers)\n        else:\n            self.rnn = nn.LSTM(embedding_size, hidden_size, nlayers, dropout=dropout)\n\n        self.hidden = None\n\n        self.nout = nout\n        if nout is not None:\n            self.proj = nn.Linear(hidden_size, nout)\n            self.initialize(self.proj.weight)\n            self.decoder = nn.Linear(nout, len(dictionary))\n        else:\n            self.proj = None\n            self.decoder = nn.Linear(hidden_size, len(dictionary))\n\n        self.init_weights()\n\n        # auto-spawn on GPU if available\n        self.to(flair.device)\n\n    def init_weights(self):\n        initrange = 0.1\n        self.encoder.weight.detach().uniform_(-initrange, initrange)\n        self.decoder.bias.detach().fill_(0)\n        self.decoder.weight.detach().uniform_(-initrange, initrange)\n\n    def set_hidden(self, hidden):\n        self.hidden = hidden\n\n    def forward(self, input, hidden, ordered_sequence_lengths=None):\n        encoded = self.encoder(input)\n        emb = self.drop(encoded)\n\n        self.rnn.flatten_parameters()\n\n        output, hidden = self.rnn(emb, hidden)\n\n        if self.proj is not None:\n            output = self.proj(output)\n\n        output = self.drop(output)\n\n        decoded = self.decoder(output.view(output.size(0) * output.size(1), output.size(2)))\n\n        return decoded.view(output.size(0), output.size(1), decoded.size(1)), output, hidden\n\n    def init_hidden(self, bsz):\n        weight = next(self.parameters()).detach()\n        return (weight.new(self.nlayers, bsz, self.hidden_size).zero_().clone().detach(),\n                weight.new(self.nlayers, bsz, self.hidden_size).zero_().clone().detach())\n\n    def get_representation(self, strings: List[str], chars_per_chunk: int = 512):\n\n        # cut up the input into chunks of max charlength = chunk_size\n        longest = len(strings[0])\n        chunks = []\n        splice_begin = 0\n        for splice_end in range(chars_per_chunk, longest, chars_per_chunk):\n            chunks.append([text[splice_begin:splice_end] for text in strings])\n            splice_begin = splice_end\n\n        chunks.append([text[splice_begin:longest] for text in strings])\n        hidden = self.init_hidden(len(chunks[0]))\n\n        output_parts = []\n\n        # push each chunk through the RNN language model\n        for chunk in chunks:\n\n            sequences_as_char_indices: List[List[int]] = []\n            for string in chunk:\n                char_indices = [self.dictionary.get_idx_for_item(char) for char in string]\n                sequences_as_char_indices.append(char_indices)\n\n            batch = torch.LongTensor(sequences_as_char_indices).transpose(0, 1)\n            batch = batch.to(flair.device)\n\n            prediction, rnn_output, hidden = self.forward(batch, hidden)\n            rnn_output = rnn_output.detach()\n\n            output_parts.append(rnn_output)\n\n        # concatenate all chunks to make final output\n        output = torch.cat(output_parts)\n\n        return output\n\n    def get_output(self, text: str):\n        char_indices = [self.dictionary.get_idx_for_item(char) for char in text]\n        input_vector = torch.LongTensor([char_indices]).transpose(0, 1)\n\n        hidden = self.init_hidden(1)\n        prediction, rnn_output, hidden = self.forward(input_vector, hidden)\n\n        return self.repackage_hidden(hidden)\n\n    def repackage_hidden(self, h):\n        \"\"\"Wraps hidden states in new Variables, to detach them from their history.\"\"\"\n        if type(h) == torch.Tensor:\n            return h.clone().detach()\n        else:\n            return tuple(self.repackage_hidden(v) for v in h)\n\n    def initialize(self, matrix):\n        in_, out_ = matrix.size()\n        stdv = math.sqrt(3. / (in_ + out_))\n        matrix.detach().uniform_(-stdv, stdv)\n\n    @classmethod\n    def load_language_model(cls, model_file: Union[Path, str]):\n\n        state = torch.load(str(model_file), map_location=flair.device)\n\n        model = LanguageModel(state['dictionary'],\n                              state['is_forward_lm'],\n                              state['hidden_size'],\n                              state['nlayers'],\n                              state['embedding_size'],\n                              state['nout'],\n                              state['dropout'])\n        model.load_state_dict(state['state_dict'])\n        model.eval()\n        model.to(flair.device)\n\n        return model\n\n    @classmethod\n    def load_checkpoint(cls, model_file: Path):\n        state = torch.load(str(model_file), map_location=flair.device)\n\n        epoch = state['epoch'] if 'epoch' in state else None\n        split = state['split'] if 'split' in state else None\n        loss = state['loss'] if 'loss' in state else None\n        optimizer_state_dict = state['optimizer_state_dict'] if 'optimizer_state_dict' in state else None\n\n        model = LanguageModel(state['dictionary'],\n                              state['is_forward_lm'],\n                              state['hidden_size'],\n                              state['nlayers'],\n                              state['embedding_size'],\n                              state['nout'],\n                              state['dropout'])\n        model.load_state_dict(state['state_dict'])\n        model.eval()\n        model.to(flair.device)\n\n        return {'model': model, 'epoch': epoch, 'split': split, 'loss': loss,\n                'optimizer_state_dict': optimizer_state_dict}\n\n    def save_checkpoint(self, file: Path, optimizer: Optimizer, epoch: int, split: int, loss: float):\n        model_state = {\n            'state_dict': self.state_dict(),\n            'dictionary': self.dictionary,\n            'is_forward_lm': self.is_forward_lm,\n            'hidden_size': self.hidden_size,\n            'nlayers': self.nlayers,\n            'embedding_size': self.embedding_size,\n            'nout': self.nout,\n            'dropout': self.dropout,\n            'optimizer_state_dict': optimizer.state_dict(),\n            'epoch': epoch,\n            'split': split,\n            'loss': loss\n        }\n\n        torch.save(model_state, str(file), pickle_protocol=4)\n\n    def save(self, file: Path):\n        model_state = {\n            'state_dict': self.state_dict(),\n            'dictionary': self.dictionary,\n            'is_forward_lm': self.is_forward_lm,\n            'hidden_size': self.hidden_size,\n            'nlayers': self.nlayers,\n            'embedding_size': self.embedding_size,\n            'nout': self.nout,\n            'dropout': self.dropout\n        }\n\n        torch.save(model_state, str(file), pickle_protocol=4)\n\n    def generate_text(self, prefix: str = '\\n', number_of_characters: int = 1000, temperature: float = 1.0,\n                      break_on_suffix=None) -> Tuple[str, float]:\n\n        if prefix == '':\n            prefix = '\\n'\n\n        with torch.no_grad():\n            characters = []\n\n            idx2item = self.dictionary.idx2item\n\n            # initial hidden state\n            hidden = self.init_hidden(1)\n\n            if len(prefix) > 1:\n\n                char_tensors = []\n                for character in prefix[:-1]:\n                    char_tensors.append(\n                        torch.tensor(self.dictionary.get_idx_for_item(character)).unsqueeze(0).unsqueeze(0))\n\n                input = torch.cat(char_tensors)\n                if torch.cuda.is_available():\n                    input = input.cuda()\n\n                prediction, _, hidden = self.forward(input, hidden)\n\n            input = torch.tensor(self.dictionary.get_idx_for_item(prefix[-1])).unsqueeze(0).unsqueeze(0)\n\n            log_prob = 0.\n\n            for i in range(number_of_characters):\n\n                if torch.cuda.is_available():\n                    input = input.cuda()\n\n                # get predicted weights\n                prediction, _, hidden = self.forward(input, hidden)\n                prediction = prediction.squeeze().detach()\n                decoder_output = prediction\n\n                # divide by temperature\n                prediction = prediction.div(temperature)\n\n                # to prevent overflow problem with small temperature values, substract largest value from all\n                # this makes a vector in which the largest value is 0\n                max = torch.max(prediction)\n                prediction -= max\n\n                # compute word weights with exponential function\n                word_weights = prediction.exp().cpu()\n\n                # try sampling multinomial distribution for next character\n                try:\n                    word_idx = torch.multinomial(word_weights, 1)[0]\n                except:\n                    word_idx = torch.tensor(0)\n\n                # print(word_idx)\n                prob = decoder_output[word_idx]\n                log_prob += prob\n\n                input = word_idx.detach().unsqueeze(0).unsqueeze(0)\n                word = idx2item[word_idx].decode('UTF-8')\n                characters.append(word)\n\n                if break_on_suffix is not None:\n                    if ''.join(characters).endswith(break_on_suffix):\n                        break\n\n            text = prefix + ''.join(characters)\n\n            log_prob = log_prob.item()\n            log_prob /= len(characters)\n\n            if not self.is_forward_lm:\n                text = text[::-1]\n\n            return text, log_prob\n\n    def calculate_perplexity(self, text: str) -> float:\n\n        if not self.is_forward_lm:\n            text = text[::-1]\n\n        # input ids\n        input = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[:-1]]).unsqueeze(1)\n\n        # push list of character IDs through model\n        hidden = self.init_hidden(1)\n        prediction, _, hidden = self.forward(input, hidden)\n\n        # the target is always the next character\n        targets = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[1:]])\n\n        # use cross entropy loss to compare output of forward pass with targets\n        cross_entroy_loss = torch.nn.CrossEntropyLoss()\n        loss = cross_entroy_loss(prediction.view(-1, len(self.dictionary)), targets).item()\n\n        # exponentiate cross-entropy loss to calculate perplexity\n        perplexity = math.exp(loss)\n\n        return perplexity\n", "path": "flair/models/language_model.py"}], "after_files": [{"content": "from pathlib import Path\n\nimport torch.nn as nn\nimport torch\nimport math\nfrom typing import Union, Tuple\nfrom typing import List\n\nfrom torch.optim import Optimizer\n\nimport flair\nfrom flair.data import Dictionary\n\n\nclass LanguageModel(nn.Module):\n    \"\"\"Container module with an encoder, a recurrent module, and a decoder.\"\"\"\n\n    def __init__(self,\n                 dictionary: Dictionary,\n                 is_forward_lm: bool,\n                 hidden_size: int,\n                 nlayers: int,\n                 embedding_size: int = 100,\n                 nout=None,\n                 dropout=0.1):\n\n        super(LanguageModel, self).__init__()\n\n        self.dictionary = dictionary\n        self.is_forward_lm: bool = is_forward_lm\n\n        self.dropout = dropout\n        self.hidden_size = hidden_size\n        self.embedding_size = embedding_size\n        self.nlayers = nlayers\n\n        self.drop = nn.Dropout(dropout)\n        self.encoder = nn.Embedding(len(dictionary), embedding_size)\n\n        if nlayers == 1:\n            self.rnn = nn.LSTM(embedding_size, hidden_size, nlayers)\n        else:\n            self.rnn = nn.LSTM(embedding_size, hidden_size, nlayers, dropout=dropout)\n\n        self.hidden = None\n\n        self.nout = nout\n        if nout is not None:\n            self.proj = nn.Linear(hidden_size, nout)\n            self.initialize(self.proj.weight)\n            self.decoder = nn.Linear(nout, len(dictionary))\n        else:\n            self.proj = None\n            self.decoder = nn.Linear(hidden_size, len(dictionary))\n\n        self.init_weights()\n\n        # auto-spawn on GPU if available\n        self.to(flair.device)\n\n    def init_weights(self):\n        initrange = 0.1\n        self.encoder.weight.detach().uniform_(-initrange, initrange)\n        self.decoder.bias.detach().fill_(0)\n        self.decoder.weight.detach().uniform_(-initrange, initrange)\n\n    def set_hidden(self, hidden):\n        self.hidden = hidden\n\n    def forward(self, input, hidden, ordered_sequence_lengths=None):\n        encoded = self.encoder(input)\n        emb = self.drop(encoded)\n\n        self.rnn.flatten_parameters()\n\n        output, hidden = self.rnn(emb, hidden)\n\n        if self.proj is not None:\n            output = self.proj(output)\n\n        output = self.drop(output)\n\n        decoded = self.decoder(output.view(output.size(0) * output.size(1), output.size(2)))\n\n        return decoded.view(output.size(0), output.size(1), decoded.size(1)), output, hidden\n\n    def init_hidden(self, bsz):\n        weight = next(self.parameters()).detach()\n        return (weight.new(self.nlayers, bsz, self.hidden_size).zero_().clone().detach(),\n                weight.new(self.nlayers, bsz, self.hidden_size).zero_().clone().detach())\n\n    def get_representation(self, strings: List[str], chars_per_chunk: int = 512):\n\n        # cut up the input into chunks of max charlength = chunk_size\n        longest = len(strings[0])\n        chunks = []\n        splice_begin = 0\n        for splice_end in range(chars_per_chunk, longest, chars_per_chunk):\n            chunks.append([text[splice_begin:splice_end] for text in strings])\n            splice_begin = splice_end\n\n        chunks.append([text[splice_begin:longest] for text in strings])\n        hidden = self.init_hidden(len(chunks[0]))\n\n        output_parts = []\n\n        # push each chunk through the RNN language model\n        for chunk in chunks:\n\n            sequences_as_char_indices: List[List[int]] = []\n            for string in chunk:\n                char_indices = [self.dictionary.get_idx_for_item(char) for char in string]\n                sequences_as_char_indices.append(char_indices)\n\n            batch = torch.LongTensor(sequences_as_char_indices).transpose(0, 1)\n            batch = batch.to(flair.device)\n\n            prediction, rnn_output, hidden = self.forward(batch, hidden)\n            rnn_output = rnn_output.detach()\n\n            output_parts.append(rnn_output)\n\n        # concatenate all chunks to make final output\n        output = torch.cat(output_parts)\n\n        return output\n\n    def get_output(self, text: str):\n        char_indices = [self.dictionary.get_idx_for_item(char) for char in text]\n        input_vector = torch.LongTensor([char_indices]).transpose(0, 1)\n\n        hidden = self.init_hidden(1)\n        prediction, rnn_output, hidden = self.forward(input_vector, hidden)\n\n        return self.repackage_hidden(hidden)\n\n    def repackage_hidden(self, h):\n        \"\"\"Wraps hidden states in new Variables, to detach them from their history.\"\"\"\n        if type(h) == torch.Tensor:\n            return h.clone().detach()\n        else:\n            return tuple(self.repackage_hidden(v) for v in h)\n\n    def initialize(self, matrix):\n        in_, out_ = matrix.size()\n        stdv = math.sqrt(3. / (in_ + out_))\n        matrix.detach().uniform_(-stdv, stdv)\n\n    @classmethod\n    def load_language_model(cls, model_file: Union[Path, str]):\n\n        state = torch.load(str(model_file), map_location=flair.device)\n\n        model = LanguageModel(state['dictionary'],\n                              state['is_forward_lm'],\n                              state['hidden_size'],\n                              state['nlayers'],\n                              state['embedding_size'],\n                              state['nout'],\n                              state['dropout'])\n        model.load_state_dict(state['state_dict'])\n        model.eval()\n        model.to(flair.device)\n\n        return model\n\n    @classmethod\n    def load_checkpoint(cls, model_file: Path):\n        state = torch.load(str(model_file), map_location=flair.device)\n\n        epoch = state['epoch'] if 'epoch' in state else None\n        split = state['split'] if 'split' in state else None\n        loss = state['loss'] if 'loss' in state else None\n        optimizer_state_dict = state['optimizer_state_dict'] if 'optimizer_state_dict' in state else None\n\n        model = LanguageModel(state['dictionary'],\n                              state['is_forward_lm'],\n                              state['hidden_size'],\n                              state['nlayers'],\n                              state['embedding_size'],\n                              state['nout'],\n                              state['dropout'])\n        model.load_state_dict(state['state_dict'])\n        model.eval()\n        model.to(flair.device)\n\n        return {'model': model, 'epoch': epoch, 'split': split, 'loss': loss,\n                'optimizer_state_dict': optimizer_state_dict}\n\n    def save_checkpoint(self, file: Path, optimizer: Optimizer, epoch: int, split: int, loss: float):\n        model_state = {\n            'state_dict': self.state_dict(),\n            'dictionary': self.dictionary,\n            'is_forward_lm': self.is_forward_lm,\n            'hidden_size': self.hidden_size,\n            'nlayers': self.nlayers,\n            'embedding_size': self.embedding_size,\n            'nout': self.nout,\n            'dropout': self.dropout,\n            'optimizer_state_dict': optimizer.state_dict(),\n            'epoch': epoch,\n            'split': split,\n            'loss': loss\n        }\n\n        torch.save(model_state, str(file), pickle_protocol=4)\n\n    def save(self, file: Path):\n        model_state = {\n            'state_dict': self.state_dict(),\n            'dictionary': self.dictionary,\n            'is_forward_lm': self.is_forward_lm,\n            'hidden_size': self.hidden_size,\n            'nlayers': self.nlayers,\n            'embedding_size': self.embedding_size,\n            'nout': self.nout,\n            'dropout': self.dropout\n        }\n\n        torch.save(model_state, str(file), pickle_protocol=4)\n\n    def generate_text(self, prefix: str = '\\n', number_of_characters: int = 1000, temperature: float = 1.0,\n                      break_on_suffix=None) -> Tuple[str, float]:\n\n        if prefix == '':\n            prefix = '\\n'\n\n        with torch.no_grad():\n            characters = []\n\n            idx2item = self.dictionary.idx2item\n\n            # initial hidden state\n            hidden = self.init_hidden(1)\n\n            if len(prefix) > 1:\n\n                char_tensors = []\n                for character in prefix[:-1]:\n                    char_tensors.append(\n                        torch.tensor(self.dictionary.get_idx_for_item(character)).unsqueeze(0).unsqueeze(0))\n\n                input = torch.cat(char_tensors)\n                if torch.cuda.is_available():\n                    input = input.cuda()\n\n                prediction, _, hidden = self.forward(input, hidden)\n\n            input = torch.tensor(self.dictionary.get_idx_for_item(prefix[-1])).unsqueeze(0).unsqueeze(0)\n\n            log_prob = 0.\n\n            for i in range(number_of_characters):\n\n                if torch.cuda.is_available():\n                    input = input.cuda()\n\n                # get predicted weights\n                prediction, _, hidden = self.forward(input, hidden)\n                prediction = prediction.squeeze().detach()\n                decoder_output = prediction\n\n                # divide by temperature\n                prediction = prediction.div(temperature)\n\n                # to prevent overflow problem with small temperature values, substract largest value from all\n                # this makes a vector in which the largest value is 0\n                max = torch.max(prediction)\n                prediction -= max\n\n                # compute word weights with exponential function\n                word_weights = prediction.exp().cpu()\n\n                # try sampling multinomial distribution for next character\n                try:\n                    word_idx = torch.multinomial(word_weights, 1)[0]\n                except:\n                    word_idx = torch.tensor(0)\n\n                # print(word_idx)\n                prob = decoder_output[word_idx]\n                log_prob += prob\n\n                input = word_idx.detach().unsqueeze(0).unsqueeze(0)\n                word = idx2item[word_idx].decode('UTF-8')\n                characters.append(word)\n\n                if break_on_suffix is not None:\n                    if ''.join(characters).endswith(break_on_suffix):\n                        break\n\n            text = prefix + ''.join(characters)\n\n            log_prob = log_prob.item()\n            log_prob /= len(characters)\n\n            if not self.is_forward_lm:\n                text = text[::-1]\n\n            return text, log_prob\n\n    def calculate_perplexity(self, text: str) -> float:\n\n        if not self.is_forward_lm:\n            text = text[::-1]\n\n        # input ids\n        input = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[:-1]]).unsqueeze(1)\n        input = input.to(flair.device)\n\n        # push list of character IDs through model\n        hidden = self.init_hidden(1)\n        prediction, _, hidden = self.forward(input, hidden)\n\n        # the target is always the next character\n        targets = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[1:]])\n        targets = targets.to(flair.device)\n\n        # use cross entropy loss to compare output of forward pass with targets\n        cross_entroy_loss = torch.nn.CrossEntropyLoss()\n        loss = cross_entroy_loss(prediction.view(-1, len(self.dictionary)), targets).item()\n\n        # exponentiate cross-entropy loss to calculate perplexity\n        perplexity = math.exp(loss)\n\n        return perplexity\n", "path": "flair/models/language_model.py"}]}