We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Documentation error in GPT2Tokenizer
Documentation page:
https://huggingface.co/transformers/model_doc/gpt2.html#transformers.GPT2Tokenizer
Code place:
https://github.com/huggingface/transformers/blob/d7d36181fdefdabadc53adf51bed4a2680f5880a/transformers/tokenization_gpt2.py#L112-L113

This phrase:
> Otherwise, this tokenizer encode and decode method will not conserve the absence of a space at the beginning of a string: tokenizer.decode(tokenizer.encode(“Hello”)) = ” Hello”

is **NOT** correct.

Actually:
> tokenizer.decode(tokenizer.encode(“Hello”)) = ”Hello”

Try this example:
```
from transformers import GPT2Tokenizer
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
print("'" + tokenizer.decode(tokenizer.encode("Hello")) + "'")
```
Output:
```
'Hello'
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `transformers/tokenization_gpt2.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 """Tokenization classes for OpenAI GPT."""
16 from __future__ import (absolute_import, division, print_function,
17                         unicode_literals)
18 
19 import sys
20 import json
21 import logging
22 import os
23 import regex as re
24 from io import open
25 
26 try:
27     from functools import lru_cache
28 except ImportError:
29     # Just a dummy decorator to get the checks to run on python2
30     # because honestly I don't want to support a byte-level unicode BPE tokenizer on python 2 right now.
31     def lru_cache():
32         return lambda func: func
33 
34 from .tokenization_utils import PreTrainedTokenizer
35 
36 logger = logging.getLogger(__name__)
37 
38 VOCAB_FILES_NAMES = {
39     'vocab_file': 'vocab.json',
40     'merges_file': 'merges.txt',
41 }
42 
43 PRETRAINED_VOCAB_FILES_MAP = {
44     'vocab_file':
45     {
46         'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json",
47         'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-vocab.json",
48         'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-vocab.json",
49         'gpt2-xl': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-vocab.json",
50         'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-vocab.json",
51     },
52     'merges_file':
53     {
54         'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt",
55         'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-merges.txt",
56         'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-merges.txt",
57         'gpt2-xl': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-merges.txt",
58         'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-merges.txt",
59     },
60 }
61 
62 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
63     'gpt2': 1024,
64     'gpt2-medium': 1024,
65     'gpt2-large': 1024,
66     'gpt2-xl': 1024,
67     'distilgpt2': 1024,
68 }
69 
70 @lru_cache()
71 def bytes_to_unicode():
72     """
73     Returns list of utf-8 byte and a mapping to unicode strings.
74     We specifically avoids mapping to whitespace/control characters the bpe code barfs on.
75     
76     The reversible bpe codes work on unicode strings.
77     This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
78     When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
79     This is a signficant percentage of your normal, say, 32K bpe vocab.
80     To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
81     """
82     _chr = unichr if sys.version_info[0] == 2 else chr
83     bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
84     cs = bs[:]
85     n = 0
86     for b in range(2**8):
87         if b not in bs:
88             bs.append(b)
89             cs.append(2**8+n)
90             n += 1
91     cs = [_chr(n) for n in cs]
92     return dict(zip(bs, cs))
93 
94 def get_pairs(word):
95     """Return set of symbol pairs in a word.
96 
97     Word is represented as tuple of symbols (symbols being variable-length strings).
98     """
99     pairs = set()
100     prev_char = word[0]
101     for char in word[1:]:
102         pairs.add((prev_char, char))
103         prev_char = char
104     return pairs
105 
106 class GPT2Tokenizer(PreTrainedTokenizer):
107     """
108     GPT-2 BPE tokenizer. Peculiarities:
109         - Byte-level Byte-Pair-Encoding
110         - Requires a space to start the input string => the encoding methods should be called with the
111           ``add_prefix_space`` flag set to ``True``.
112           Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve
113           the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"`
114     """
115     vocab_files_names = VOCAB_FILES_NAMES
116     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
117     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
118 
119     def __init__(self, vocab_file, merges_file, errors='replace', unk_token="<|endoftext|>",
120                  bos_token="<|endoftext|>", eos_token="<|endoftext|>", **kwargs):
121         super(GPT2Tokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
122         self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens
123         self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens
124 
125         self.encoder = json.load(open(vocab_file, encoding="utf-8"))
126         self.decoder = {v: k for k, v in self.encoder.items()}
127         self.errors = errors  # how to handle errors in decoding
128         self.byte_encoder = bytes_to_unicode()
129         self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
130         bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1]
131         bpe_merges = [tuple(merge.split()) for merge in bpe_data]
132         self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
133         self.cache = {}
134 
135         # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
136         self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
137 
138     @property
139     def vocab_size(self):
140         return len(self.encoder)
141 
142     def bpe(self, token):
143         if token in self.cache:
144             return self.cache[token]
145         word = tuple(token)
146         pairs = get_pairs(word)
147 
148         if not pairs:
149             return token
150 
151         while True:
152             bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
153             if bigram not in self.bpe_ranks:
154                 break
155             first, second = bigram
156             new_word = []
157             i = 0
158             while i < len(word):
159                 try:
160                     j = word.index(first, i)
161                     new_word.extend(word[i:j])
162                     i = j
163                 except:
164                     new_word.extend(word[i:])
165                     break
166 
167                 if word[i] == first and i < len(word)-1 and word[i+1] == second:
168                     new_word.append(first+second)
169                     i += 2
170                 else:
171                     new_word.append(word[i])
172                     i += 1
173             new_word = tuple(new_word)
174             word = new_word
175             if len(word) == 1:
176                 break
177             else:
178                 pairs = get_pairs(word)
179         word = ' '.join(word)
180         self.cache[token] = word
181         return word
182 
183     def _tokenize(self, text, add_prefix_space=False):
184         """ Tokenize a string.
185             Args:
186                 - add_prefix_space (boolean, default False):
187                     Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers.
188         """
189         if add_prefix_space:
190             text = ' ' + text
191 
192         bpe_tokens = []
193         for token in re.findall(self.pat, text):
194             if sys.version_info[0] == 2:
195                 token = ''.join(self.byte_encoder[ord(b)] for b in token) # Maps all our bytes to unicode strings, avoiding controle tokens of the BPE (spaces in our case)
196             else:
197                 token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) # Maps all our bytes to unicode strings, avoiding controle tokens of the BPE (spaces in our case)
198             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(' '))
199         return bpe_tokens
200 
201     def _convert_token_to_id(self, token):
202         """ Converts a token (str/unicode) in an id using the vocab. """
203         return self.encoder.get(token, self.encoder.get(self.unk_token))
204 
205     def _convert_id_to_token(self, index):
206         """Converts an index (integer) in a token (string/unicode) using the vocab."""
207         return self.decoder.get(index)
208 
209     def convert_tokens_to_string(self, tokens):
210         """ Converts a sequence of tokens (string) in a single string. """
211         text = ''.join(tokens)
212         text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
213         return text
214 
215     def save_vocabulary(self, save_directory):
216         """Save the tokenizer vocabulary and merge files to a directory."""
217         if not os.path.isdir(save_directory):
218             logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
219             return
220         vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])
221         merge_file = os.path.join(save_directory, VOCAB_FILES_NAMES['merges_file'])
222 
223         with open(vocab_file, 'w', encoding='utf-8') as f:
224             f.write(json.dumps(self.encoder, ensure_ascii=False))
225 
226         index = 0
227         with open(merge_file, "w", encoding="utf-8") as writer:
228             writer.write(u'#version: 0.2\n')
229             for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
230                 if index != token_index:
231                     logger.warning("Saving vocabulary to {}: BPE merge indices are not consecutive."
232                                    " Please check that the tokenizer is not corrupted!".format(merge_file))
233                     index = token_index
234                 writer.write(' '.join(bpe_tokens) + u'\n')
235                 index += 1
236 
237         return vocab_file, merge_file
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py
--- a/transformers/tokenization_gpt2.py
+++ b/transformers/tokenization_gpt2.py
@@ -107,10 +107,10 @@
     """
     GPT-2 BPE tokenizer. Peculiarities:
         - Byte-level Byte-Pair-Encoding
-        - Requires a space to start the input string => the encoding methods should be called with the
+        - Requires a space to start the input string => the encoding and tokenize methods should be called with the
           ``add_prefix_space`` flag set to ``True``.
-          Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve
-          the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"`
+          Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve
+          the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(" Hello")) = "Hello"`
     """
     vocab_files_names = VOCAB_FILES_NAMES
     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP

{"golden_diff": "diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py\n--- a/transformers/tokenization_gpt2.py\n+++ b/transformers/tokenization_gpt2.py\n@@ -107,10 +107,10 @@\n     \"\"\"\n     GPT-2 BPE tokenizer. Peculiarities:\n         - Byte-level Byte-Pair-Encoding\n-        - Requires a space to start the input string => the encoding methods should be called with the\n+        - Requires a space to start the input string => the encoding and tokenize methods should be called with the\n           ``add_prefix_space`` flag set to ``True``.\n-          Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve\n-          the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode(\"Hello\")) = \" Hello\"`\n+          Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve\n+          the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(\" Hello\")) = \"Hello\"`\n     \"\"\"\n     vocab_files_names = VOCAB_FILES_NAMES\n     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP\n", "issue": "Documentation error in GPT2Tokenizer\nDocumentation page:\r\nhttps://huggingface.co/transformers/model_doc/gpt2.html#transformers.GPT2Tokenizer\r\nCode place:\r\nhttps://github.com/huggingface/transformers/blob/d7d36181fdefdabadc53adf51bed4a2680f5880a/transformers/tokenization_gpt2.py#L112-L113\r\n\r\nThis phrase:\r\n> Otherwise, this tokenizer encode and decode method will not conserve the absence of a space at the beginning of a string: tokenizer.decode(tokenizer.encode(\u201cHello\u201d)) = \u201d Hello\u201d\r\n\r\nis **NOT** correct.\r\n\r\nActually:\r\n> tokenizer.decode(tokenizer.encode(\u201cHello\u201d)) = \u201dHello\u201d\r\n\r\nTry this example:\r\n```\r\nfrom transformers import GPT2Tokenizer\r\ntokenizer = GPT2Tokenizer.from_pretrained('gpt2')\r\nprint(\"'\" + tokenizer.decode(tokenizer.encode(\"Hello\")) + \"'\")\r\n```\r\nOutput:\r\n```\r\n'Hello'\r\n```\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Tokenization classes for OpenAI GPT.\"\"\"\nfrom __future__ import (absolute_import, division, print_function,\n                        unicode_literals)\n\nimport sys\nimport json\nimport logging\nimport os\nimport regex as re\nfrom io import open\n\ntry:\n    from functools import lru_cache\nexcept ImportError:\n    # Just a dummy decorator to get the checks to run on python2\n    # because honestly I don't want to support a byte-level unicode BPE tokenizer on python 2 right now.\n    def lru_cache():\n        return lambda func: func\n\nfrom .tokenization_utils import PreTrainedTokenizer\n\nlogger = logging.getLogger(__name__)\n\nVOCAB_FILES_NAMES = {\n    'vocab_file': 'vocab.json',\n    'merges_file': 'merges.txt',\n}\n\nPRETRAINED_VOCAB_FILES_MAP = {\n    'vocab_file':\n    {\n        'gpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json\",\n        'gpt2-medium': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-vocab.json\",\n        'gpt2-large': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-vocab.json\",\n        'gpt2-xl': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-vocab.json\",\n        'distilgpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-vocab.json\",\n    },\n    'merges_file':\n    {\n        'gpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt\",\n        'gpt2-medium': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-merges.txt\",\n        'gpt2-large': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-merges.txt\",\n        'gpt2-xl': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-merges.txt\",\n        'distilgpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-merges.txt\",\n    },\n}\n\nPRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {\n    'gpt2': 1024,\n    'gpt2-medium': 1024,\n    'gpt2-large': 1024,\n    'gpt2-xl': 1024,\n    'distilgpt2': 1024,\n}\n\n@lru_cache()\ndef bytes_to_unicode():\n    \"\"\"\n    Returns list of utf-8 byte and a mapping to unicode strings.\n    We specifically avoids mapping to whitespace/control characters the bpe code barfs on.\n    \n    The reversible bpe codes work on unicode strings.\n    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n    This is a signficant percentage of your normal, say, 32K bpe vocab.\n    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n    \"\"\"\n    _chr = unichr if sys.version_info[0] == 2 else chr\n    bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"\u00a1\"), ord(\"\u00ac\")+1))+list(range(ord(\"\u00ae\"), ord(\"\u00ff\")+1))\n    cs = bs[:]\n    n = 0\n    for b in range(2**8):\n        if b not in bs:\n            bs.append(b)\n            cs.append(2**8+n)\n            n += 1\n    cs = [_chr(n) for n in cs]\n    return dict(zip(bs, cs))\n\ndef get_pairs(word):\n    \"\"\"Return set of symbol pairs in a word.\n\n    Word is represented as tuple of symbols (symbols being variable-length strings).\n    \"\"\"\n    pairs = set()\n    prev_char = word[0]\n    for char in word[1:]:\n        pairs.add((prev_char, char))\n        prev_char = char\n    return pairs\n\nclass GPT2Tokenizer(PreTrainedTokenizer):\n    \"\"\"\n    GPT-2 BPE tokenizer. Peculiarities:\n        - Byte-level Byte-Pair-Encoding\n        - Requires a space to start the input string => the encoding methods should be called with the\n          ``add_prefix_space`` flag set to ``True``.\n          Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve\n          the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode(\"Hello\")) = \" Hello\"`\n    \"\"\"\n    vocab_files_names = VOCAB_FILES_NAMES\n    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP\n    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES\n\n    def __init__(self, vocab_file, merges_file, errors='replace', unk_token=\"<|endoftext|>\",\n                 bos_token=\"<|endoftext|>\", eos_token=\"<|endoftext|>\", **kwargs):\n        super(GPT2Tokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)\n        self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens\n        self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens\n\n        self.encoder = json.load(open(vocab_file, encoding=\"utf-8\"))\n        self.decoder = {v: k for k, v in self.encoder.items()}\n        self.errors = errors  # how to handle errors in decoding\n        self.byte_encoder = bytes_to_unicode()\n        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}\n        bpe_data = open(merges_file, encoding='utf-8').read().split('\\n')[1:-1]\n        bpe_merges = [tuple(merge.split()) for merge in bpe_data]\n        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))\n        self.cache = {}\n\n        # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions\n        self.pat = re.compile(r\"\"\"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\"\"\")\n\n    @property\n    def vocab_size(self):\n        return len(self.encoder)\n\n    def bpe(self, token):\n        if token in self.cache:\n            return self.cache[token]\n        word = tuple(token)\n        pairs = get_pairs(word)\n\n        if not pairs:\n            return token\n\n        while True:\n            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))\n            if bigram not in self.bpe_ranks:\n                break\n            first, second = bigram\n            new_word = []\n            i = 0\n            while i < len(word):\n                try:\n                    j = word.index(first, i)\n                    new_word.extend(word[i:j])\n                    i = j\n                except:\n                    new_word.extend(word[i:])\n                    break\n\n                if word[i] == first and i < len(word)-1 and word[i+1] == second:\n                    new_word.append(first+second)\n                    i += 2\n                else:\n                    new_word.append(word[i])\n                    i += 1\n            new_word = tuple(new_word)\n            word = new_word\n            if len(word) == 1:\n                break\n            else:\n                pairs = get_pairs(word)\n        word = ' '.join(word)\n        self.cache[token] = word\n        return word\n\n    def _tokenize(self, text, add_prefix_space=False):\n        \"\"\" Tokenize a string.\n            Args:\n                - add_prefix_space (boolean, default False):\n                    Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers.\n        \"\"\"\n        if add_prefix_space:\n            text = ' ' + text\n\n        bpe_tokens = []\n        for token in re.findall(self.pat, text):\n            if sys.version_info[0] == 2:\n                token = ''.join(self.byte_encoder[ord(b)] for b in token) # Maps all our bytes to unicode strings, avoiding controle tokens of the BPE (spaces in our case)\n            else:\n                token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) # Maps all our bytes to unicode strings, avoiding controle tokens of the BPE (spaces in our case)\n            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(' '))\n        return bpe_tokens\n\n    def _convert_token_to_id(self, token):\n        \"\"\" Converts a token (str/unicode) in an id using the vocab. \"\"\"\n        return self.encoder.get(token, self.encoder.get(self.unk_token))\n\n    def _convert_id_to_token(self, index):\n        \"\"\"Converts an index (integer) in a token (string/unicode) using the vocab.\"\"\"\n        return self.decoder.get(index)\n\n    def convert_tokens_to_string(self, tokens):\n        \"\"\" Converts a sequence of tokens (string) in a single string. \"\"\"\n        text = ''.join(tokens)\n        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)\n        return text\n\n    def save_vocabulary(self, save_directory):\n        \"\"\"Save the tokenizer vocabulary and merge files to a directory.\"\"\"\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])\n        merge_file = os.path.join(save_directory, VOCAB_FILES_NAMES['merges_file'])\n\n        with open(vocab_file, 'w', encoding='utf-8') as f:\n            f.write(json.dumps(self.encoder, ensure_ascii=False))\n\n        index = 0\n        with open(merge_file, \"w\", encoding=\"utf-8\") as writer:\n            writer.write(u'#version: 0.2\\n')\n            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):\n                if index != token_index:\n                    logger.warning(\"Saving vocabulary to {}: BPE merge indices are not consecutive.\"\n                                   \" Please check that the tokenizer is not corrupted!\".format(merge_file))\n                    index = token_index\n                writer.write(' '.join(bpe_tokens) + u'\\n')\n                index += 1\n\n        return vocab_file, merge_file", "path": "transformers/tokenization_gpt2.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Tokenization classes for OpenAI GPT.\"\"\"\nfrom __future__ import (absolute_import, division, print_function,\n                        unicode_literals)\n\nimport sys\nimport json\nimport logging\nimport os\nimport regex as re\nfrom io import open\n\ntry:\n    from functools import lru_cache\nexcept ImportError:\n    # Just a dummy decorator to get the checks to run on python2\n    # because honestly I don't want to support a byte-level unicode BPE tokenizer on python 2 right now.\n    def lru_cache():\n        return lambda func: func\n\nfrom .tokenization_utils import PreTrainedTokenizer\n\nlogger = logging.getLogger(__name__)\n\nVOCAB_FILES_NAMES = {\n    'vocab_file': 'vocab.json',\n    'merges_file': 'merges.txt',\n}\n\nPRETRAINED_VOCAB_FILES_MAP = {\n    'vocab_file':\n    {\n        'gpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json\",\n        'gpt2-medium': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-vocab.json\",\n        'gpt2-large': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-vocab.json\",\n        'gpt2-xl': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-vocab.json\",\n        'distilgpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-vocab.json\",\n    },\n    'merges_file':\n    {\n        'gpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt\",\n        'gpt2-medium': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-merges.txt\",\n        'gpt2-large': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-merges.txt\",\n        'gpt2-xl': \"https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-merges.txt\",\n        'distilgpt2': \"https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-merges.txt\",\n    },\n}\n\nPRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {\n    'gpt2': 1024,\n    'gpt2-medium': 1024,\n    'gpt2-large': 1024,\n    'gpt2-xl': 1024,\n    'distilgpt2': 1024,\n}\n\n@lru_cache()\ndef bytes_to_unicode():\n    \"\"\"\n    Returns list of utf-8 byte and a mapping to unicode strings.\n    We specifically avoids mapping to whitespace/control characters the bpe code barfs on.\n    \n    The reversible bpe codes work on unicode strings.\n    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n    This is a signficant percentage of your normal, say, 32K bpe vocab.\n    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n    \"\"\"\n    _chr = unichr if sys.version_info[0] == 2 else chr\n    bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"\u00a1\"), ord(\"\u00ac\")+1))+list(range(ord(\"\u00ae\"), ord(\"\u00ff\")+1))\n    cs = bs[:]\n    n = 0\n    for b in range(2**8):\n        if b not in bs:\n            bs.append(b)\n            cs.append(2**8+n)\n            n += 1\n    cs = [_chr(n) for n in cs]\n    return dict(zip(bs, cs))\n\ndef get_pairs(word):\n    \"\"\"Return set of symbol pairs in a word.\n\n    Word is represented as tuple of symbols (symbols being variable-length strings).\n    \"\"\"\n    pairs = set()\n    prev_char = word[0]\n    for char in word[1:]:\n        pairs.add((prev_char, char))\n        prev_char = char\n    return pairs\n\nclass GPT2Tokenizer(PreTrainedTokenizer):\n    \"\"\"\n    GPT-2 BPE tokenizer. Peculiarities:\n        - Byte-level Byte-Pair-Encoding\n        - Requires a space to start the input string => the encoding and tokenize methods should be called with the\n          ``add_prefix_space`` flag set to ``True``.\n          Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve\n          the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(\" Hello\")) = \"Hello\"`\n    \"\"\"\n    vocab_files_names = VOCAB_FILES_NAMES\n    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP\n    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES\n\n    def __init__(self, vocab_file, merges_file, errors='replace', unk_token=\"<|endoftext|>\",\n                 bos_token=\"<|endoftext|>\", eos_token=\"<|endoftext|>\", **kwargs):\n        super(GPT2Tokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)\n        self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens\n        self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens\n\n        self.encoder = json.load(open(vocab_file, encoding=\"utf-8\"))\n        self.decoder = {v: k for k, v in self.encoder.items()}\n        self.errors = errors  # how to handle errors in decoding\n        self.byte_encoder = bytes_to_unicode()\n        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}\n        bpe_data = open(merges_file, encoding='utf-8').read().split('\\n')[1:-1]\n        bpe_merges = [tuple(merge.split()) for merge in bpe_data]\n        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))\n        self.cache = {}\n\n        # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions\n        self.pat = re.compile(r\"\"\"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\"\"\")\n\n    @property\n    def vocab_size(self):\n        return len(self.encoder)\n\n    def bpe(self, token):\n        if token in self.cache:\n            return self.cache[token]\n        word = tuple(token)\n        pairs = get_pairs(word)\n\n        if not pairs:\n            return token\n\n        while True:\n            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))\n            if bigram not in self.bpe_ranks:\n                break\n            first, second = bigram\n            new_word = []\n            i = 0\n            while i < len(word):\n                try:\n                    j = word.index(first, i)\n                    new_word.extend(word[i:j])\n                    i = j\n                except:\n                    new_word.extend(word[i:])\n                    break\n\n                if word[i] == first and i < len(word)-1 and word[i+1] == second:\n                    new_word.append(first+second)\n                    i += 2\n                else:\n                    new_word.append(word[i])\n                    i += 1\n            new_word = tuple(new_word)\n            word = new_word\n            if len(word) == 1:\n                break\n            else:\n                pairs = get_pairs(word)\n        word = ' '.join(word)\n        self.cache[token] = word\n        return word\n\n    def _tokenize(self, text, add_prefix_space=False):\n        \"\"\" Tokenize a string.\n            Args:\n                - add_prefix_space (boolean, default False):\n                    Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers.\n        \"\"\"\n        if add_prefix_space:\n            text = ' ' + text\n\n        bpe_tokens = []\n        for token in re.findall(self.pat, text):\n            if sys.version_info[0] == 2:\n                token = ''.join(self.byte_encoder[ord(b)] for b in token) # Maps all our bytes to unicode strings, avoiding controle tokens of the BPE (spaces in our case)\n            else:\n                token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) # Maps all our bytes to unicode strings, avoiding controle tokens of the BPE (spaces in our case)\n            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(' '))\n        return bpe_tokens\n\n    def _convert_token_to_id(self, token):\n        \"\"\" Converts a token (str/unicode) in an id using the vocab. \"\"\"\n        return self.encoder.get(token, self.encoder.get(self.unk_token))\n\n    def _convert_id_to_token(self, index):\n        \"\"\"Converts an index (integer) in a token (string/unicode) using the vocab.\"\"\"\n        return self.decoder.get(index)\n\n    def convert_tokens_to_string(self, tokens):\n        \"\"\" Converts a sequence of tokens (string) in a single string. \"\"\"\n        text = ''.join(tokens)\n        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)\n        return text\n\n    def save_vocabulary(self, save_directory):\n        \"\"\"Save the tokenizer vocabulary and merge files to a directory.\"\"\"\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])\n        merge_file = os.path.join(save_directory, VOCAB_FILES_NAMES['merges_file'])\n\n        with open(vocab_file, 'w', encoding='utf-8') as f:\n            f.write(json.dumps(self.encoder, ensure_ascii=False))\n\n        index = 0\n        with open(merge_file, \"w\", encoding=\"utf-8\") as writer:\n            writer.write(u'#version: 0.2\\n')\n            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):\n                if index != token_index:\n                    logger.warning(\"Saving vocabulary to {}: BPE merge indices are not consecutive.\"\n                                   \" Please check that the tokenizer is not corrupted!\".format(merge_file))\n                    index = token_index\n                writer.write(' '.join(bpe_tokens) + u'\\n')\n                index += 1\n\n        return vocab_file, merge_file", "path": "transformers/tokenization_gpt2.py"}]}