We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[CamemBert] Tokenizer function add_tokens doesn't work
## ❓ Questions & Help

Hi, 

I am trying to add new tokens to the CamemBert tokenizer, but when I run the function tokenizer.add_tokens, it doesn't seem to add any token at all :

`from transformers import CamembertTokenizer`
`tokenizer = CamembertTokenizer.from_pretrained('camembert-base')`
`tokenizer.add_tokens(['notfrenchword'])`

`Out[12]: 0`

Whereas with Bert model it works perfectly. Is this a bug or am I doing something wrong ?

Thanks


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `transformers/tokenization_camembert.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License
15 """ Tokenization classes for Camembert model."""
16 from __future__ import (absolute_import, division, print_function,
17                         unicode_literals)
18 
19 import logging
20 import os
21 from shutil import copyfile
22 
23 import sentencepiece as spm
24 from transformers.tokenization_utils import PreTrainedTokenizer
25 
26 logger = logging.getLogger(__name__)
27 
28 VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'}
29 
30 PRETRAINED_VOCAB_FILES_MAP = {
31     'vocab_file':
32     {
33     'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model",
34     }
35 }
36 
37 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
38     'camembert-base': None,
39 }
40 
41 class CamembertTokenizer(PreTrainedTokenizer):
42     """
43         Adapted from RobertaTokenizer and XLNetTokenizer
44         SentencePiece based tokenizer. Peculiarities:
45 
46             - requires `SentencePiece <https://github.com/google/sentencepiece>`_
47     """
48     vocab_files_names = VOCAB_FILES_NAMES
49     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
50     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
51 
52     def __init__(self, vocab_file, bos_token="<s>", eos_token="</s>", sep_token="</s>",
53                  cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>',
54                  additional_special_tokens=['<s>NOTUSED', '<s>NOTUSED'], **kwargs):
55         super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,
56                                                  sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,
57                                                  mask_token=mask_token, additional_special_tokens=additional_special_tokens,
58                                                  **kwargs)
59         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
60         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens
61         self.sp_model = spm.SentencePieceProcessor()
62         self.sp_model.Load(str(vocab_file))
63         self.vocab_file = vocab_file
64         # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
65         # sentencepiece vocabulary (this is the case for <s> and </s>
66         self.fairseq_tokens_to_ids = {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}
67         self.fairseq_offset = len(self.fairseq_tokens_to_ids)
68         self.fairseq_tokens_to_ids['<mask>'] = len(self.sp_model) + len(self.fairseq_tokens_to_ids)
69         self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}
70 
71     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
72         """
73         Build model inputs from a sequence or a pair of sequence for sequence classification tasks
74         by concatenating and adding special tokens.
75         A RoBERTa sequence has the following format:
76             single sequence: <s> X </s>
77             pair of sequences: <s> A </s></s> B </s>
78         """
79         if token_ids_1 is None:
80             return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
81         cls = [self.cls_token_id]
82         sep = [self.sep_token_id]
83         return cls + token_ids_0 + sep + sep + token_ids_1 + sep
84 
85     def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
86         """
87         Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
88         special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
89 
90         Args:
91             token_ids_0: list of ids (must not contain special tokens)
92             token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids
93                 for sequence pairs
94             already_has_special_tokens: (default False) Set to True if the token list is already formated with
95                 special tokens for the model
96 
97         Returns:
98             A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
99         """
100         if already_has_special_tokens:
101             if token_ids_1 is not None:
102                 raise ValueError("You should not supply a second sequence if the provided sequence of "
103                                  "ids is already formated with special tokens for the model.")
104             return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
105 
106         if token_ids_1 is None:
107             return [1] + ([0] * len(token_ids_0)) + [1]
108         return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
109 
110     def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
111         """
112         Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
113         A RoBERTa sequence pair mask has the following format:
114         0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
115         | first sequence    | second sequence
116 
117         if token_ids_1 is None, only returns the first portion of the mask (0's).
118         """
119         sep = [self.sep_token_id]
120         cls = [self.cls_token_id]
121 
122         if token_ids_1 is None:
123             return len(cls + token_ids_0 + sep) * [0]
124         return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1]
125 
126     @property
127     def vocab_size(self):
128         return self.fairseq_offset + len(self.sp_model)
129 
130     def _tokenize(self, text):
131         return self.sp_model.EncodeAsPieces(text)
132 
133     def _convert_token_to_id(self, token):
134         """ Converts a token (str/unicode) in an id using the vocab. """
135         if token in self.fairseq_tokens_to_ids:
136             return self.fairseq_tokens_to_ids[token]
137         return self.fairseq_offset + self.sp_model.PieceToId(token)
138 
139     def _convert_id_to_token(self, index):
140         """Converts an index (integer) in a token (string/unicode) using the vocab."""
141         if index in self.fairseq_ids_to_tokens:
142             return self.fairseq_ids_to_tokens[index]
143         return self.sp_model.IdToPiece(index - self.fairseq_offset)
144 
145     def save_vocabulary(self, save_directory):
146         """ Save the sentencepiece vocabulary (copy original file) and special tokens file
147             to a directory.
148         """
149         if not os.path.isdir(save_directory):
150             logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
151             return
152         out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])
153 
154         if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
155             copyfile(self.vocab_file, out_vocab_file)
156 
157         return (out_vocab_file,)
158
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
--- a/transformers/tokenization_camembert.py
+++ b/transformers/tokenization_camembert.py
@@ -51,7 +51,7 @@
 
     def __init__(self, vocab_file, bos_token="<s>", eos_token="</s>", sep_token="</s>",
                  cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>',
-                 additional_special_tokens=['<s>NOTUSED', '<s>NOTUSED'], **kwargs):
+                 additional_special_tokens=['<s>NOTUSED', '</s>NOTUSED'], **kwargs):
         super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,
                                                  sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,
                                                  mask_token=mask_token, additional_special_tokens=additional_special_tokens,
@@ -125,7 +125,7 @@
 
     @property
     def vocab_size(self):
-        return self.fairseq_offset + len(self.sp_model)
+        return len(self.fairseq_tokens_to_ids) + len(self.sp_model)
 
     def _tokenize(self, text):
         return self.sp_model.EncodeAsPieces(text)
@@ -134,6 +134,9 @@
         """ Converts a token (str/unicode) in an id using the vocab. """
         if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
+        elif self.sp_model.PieceToId(token) == 0:
+            # Convert sentence piece unk token to fairseq unk token index
+            return self.unk_token_id
         return self.fairseq_offset + self.sp_model.PieceToId(token)
 
     def _convert_id_to_token(self, index):

{"golden_diff": "diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py\n--- a/transformers/tokenization_camembert.py\n+++ b/transformers/tokenization_camembert.py\n@@ -51,7 +51,7 @@\n \n     def __init__(self, vocab_file, bos_token=\"<s>\", eos_token=\"</s>\", sep_token=\"</s>\",\n                  cls_token=\"<s>\", unk_token=\"<unk>\", pad_token='<pad>', mask_token='<mask>',\n-                 additional_special_tokens=['<s>NOTUSED', '<s>NOTUSED'], **kwargs):\n+                 additional_special_tokens=['<s>NOTUSED', '</s>NOTUSED'], **kwargs):\n         super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,\n                                                  sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,\n                                                  mask_token=mask_token, additional_special_tokens=additional_special_tokens,\n@@ -125,7 +125,7 @@\n \n     @property\n     def vocab_size(self):\n-        return self.fairseq_offset + len(self.sp_model)\n+        return len(self.fairseq_tokens_to_ids) + len(self.sp_model)\n \n     def _tokenize(self, text):\n         return self.sp_model.EncodeAsPieces(text)\n@@ -134,6 +134,9 @@\n         \"\"\" Converts a token (str/unicode) in an id using the vocab. \"\"\"\n         if token in self.fairseq_tokens_to_ids:\n             return self.fairseq_tokens_to_ids[token]\n+        elif self.sp_model.PieceToId(token) == 0:\n+            # Convert sentence piece unk token to fairseq unk token index\n+            return self.unk_token_id\n         return self.fairseq_offset + self.sp_model.PieceToId(token)\n \n     def _convert_id_to_token(self, index):\n", "issue": "[CamemBert] Tokenizer function add_tokens doesn't work\n## \u2753 Questions & Help\r\n\r\nHi, \r\n\r\nI am trying to add new tokens to the CamemBert tokenizer, but when I run the function tokenizer.add_tokens, it doesn't seem to add any token at all :\r\n\r\n`from transformers import CamembertTokenizer`\r\n`tokenizer = CamembertTokenizer.from_pretrained('camembert-base')`\r\n`tokenizer.add_tokens(['notfrenchword'])`\r\n\r\n`Out[12]: 0`\r\n\r\nWhereas with Bert model it works perfectly. Is this a bug or am I doing something wrong ?\r\n\r\nThanks\r\n\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License\n\"\"\" Tokenization classes for Camembert model.\"\"\"\nfrom __future__ import (absolute_import, division, print_function,\n                        unicode_literals)\n\nimport logging\nimport os\nfrom shutil import copyfile\n\nimport sentencepiece as spm\nfrom transformers.tokenization_utils import PreTrainedTokenizer\n\nlogger = logging.getLogger(__name__)\n\nVOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'}\n\nPRETRAINED_VOCAB_FILES_MAP = {\n    'vocab_file':\n    {\n    'camembert-base': \"https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model\",\n    }\n}\n\nPRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {\n    'camembert-base': None,\n}\n\nclass CamembertTokenizer(PreTrainedTokenizer):\n    \"\"\"\n        Adapted from RobertaTokenizer and XLNetTokenizer\n        SentencePiece based tokenizer. Peculiarities:\n\n            - requires `SentencePiece <https://github.com/google/sentencepiece>`_\n    \"\"\"\n    vocab_files_names = VOCAB_FILES_NAMES\n    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP\n    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES\n\n    def __init__(self, vocab_file, bos_token=\"<s>\", eos_token=\"</s>\", sep_token=\"</s>\",\n                 cls_token=\"<s>\", unk_token=\"<unk>\", pad_token='<pad>', mask_token='<mask>',\n                 additional_special_tokens=['<s>NOTUSED', '<s>NOTUSED'], **kwargs):\n        super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,\n                                                 sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,\n                                                 mask_token=mask_token, additional_special_tokens=additional_special_tokens,\n                                                 **kwargs)\n        self.max_len_single_sentence = self.max_len - 2  # take into account special tokens\n        self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens\n        self.sp_model = spm.SentencePieceProcessor()\n        self.sp_model.Load(str(vocab_file))\n        self.vocab_file = vocab_file\n        # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual\n        # sentencepiece vocabulary (this is the case for <s> and </s>\n        self.fairseq_tokens_to_ids = {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}\n        self.fairseq_offset = len(self.fairseq_tokens_to_ids)\n        self.fairseq_tokens_to_ids['<mask>'] = len(self.sp_model) + len(self.fairseq_tokens_to_ids)\n        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}\n\n    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):\n        \"\"\"\n        Build model inputs from a sequence or a pair of sequence for sequence classification tasks\n        by concatenating and adding special tokens.\n        A RoBERTa sequence has the following format:\n            single sequence: <s> X </s>\n            pair of sequences: <s> A </s></s> B </s>\n        \"\"\"\n        if token_ids_1 is None:\n            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]\n        cls = [self.cls_token_id]\n        sep = [self.sep_token_id]\n        return cls + token_ids_0 + sep + sep + token_ids_1 + sep\n\n    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):\n        \"\"\"\n        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding\n        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.\n\n        Args:\n            token_ids_0: list of ids (must not contain special tokens)\n            token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids\n                for sequence pairs\n            already_has_special_tokens: (default False) Set to True if the token list is already formated with\n                special tokens for the model\n\n        Returns:\n            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.\n        \"\"\"\n        if already_has_special_tokens:\n            if token_ids_1 is not None:\n                raise ValueError(\"You should not supply a second sequence if the provided sequence of \"\n                                 \"ids is already formated with special tokens for the model.\")\n            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))\n\n        if token_ids_1 is None:\n            return [1] + ([0] * len(token_ids_0)) + [1]\n        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]\n\n    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):\n        \"\"\"\n        Creates a mask from the two sequences passed to be used in a sequence-pair classification task.\n        A RoBERTa sequence pair mask has the following format:\n        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1\n        | first sequence    | second sequence\n\n        if token_ids_1 is None, only returns the first portion of the mask (0's).\n        \"\"\"\n        sep = [self.sep_token_id]\n        cls = [self.cls_token_id]\n\n        if token_ids_1 is None:\n            return len(cls + token_ids_0 + sep) * [0]\n        return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1]\n\n    @property\n    def vocab_size(self):\n        return self.fairseq_offset + len(self.sp_model)\n\n    def _tokenize(self, text):\n        return self.sp_model.EncodeAsPieces(text)\n\n    def _convert_token_to_id(self, token):\n        \"\"\" Converts a token (str/unicode) in an id using the vocab. \"\"\"\n        if token in self.fairseq_tokens_to_ids:\n            return self.fairseq_tokens_to_ids[token]\n        return self.fairseq_offset + self.sp_model.PieceToId(token)\n\n    def _convert_id_to_token(self, index):\n        \"\"\"Converts an index (integer) in a token (string/unicode) using the vocab.\"\"\"\n        if index in self.fairseq_ids_to_tokens:\n            return self.fairseq_ids_to_tokens[index]\n        return self.sp_model.IdToPiece(index - self.fairseq_offset)\n\n    def save_vocabulary(self, save_directory):\n        \"\"\" Save the sentencepiece vocabulary (copy original file) and special tokens file\n            to a directory.\n        \"\"\"\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])\n\n        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):\n            copyfile(self.vocab_file, out_vocab_file)\n\n        return (out_vocab_file,)\n", "path": "transformers/tokenization_camembert.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License\n\"\"\" Tokenization classes for Camembert model.\"\"\"\nfrom __future__ import (absolute_import, division, print_function,\n                        unicode_literals)\n\nimport logging\nimport os\nfrom shutil import copyfile\n\nimport sentencepiece as spm\nfrom transformers.tokenization_utils import PreTrainedTokenizer\n\nlogger = logging.getLogger(__name__)\n\nVOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'}\n\nPRETRAINED_VOCAB_FILES_MAP = {\n    'vocab_file':\n    {\n    'camembert-base': \"https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model\",\n    }\n}\n\nPRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {\n    'camembert-base': None,\n}\n\nclass CamembertTokenizer(PreTrainedTokenizer):\n    \"\"\"\n        Adapted from RobertaTokenizer and XLNetTokenizer\n        SentencePiece based tokenizer. Peculiarities:\n\n            - requires `SentencePiece <https://github.com/google/sentencepiece>`_\n    \"\"\"\n    vocab_files_names = VOCAB_FILES_NAMES\n    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP\n    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES\n\n    def __init__(self, vocab_file, bos_token=\"<s>\", eos_token=\"</s>\", sep_token=\"</s>\",\n                 cls_token=\"<s>\", unk_token=\"<unk>\", pad_token='<pad>', mask_token='<mask>',\n                 additional_special_tokens=['<s>NOTUSED', '</s>NOTUSED'], **kwargs):\n        super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,\n                                                 sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,\n                                                 mask_token=mask_token, additional_special_tokens=additional_special_tokens,\n                                                 **kwargs)\n        self.max_len_single_sentence = self.max_len - 2  # take into account special tokens\n        self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens\n        self.sp_model = spm.SentencePieceProcessor()\n        self.sp_model.Load(str(vocab_file))\n        self.vocab_file = vocab_file\n        # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual\n        # sentencepiece vocabulary (this is the case for <s> and </s>\n        self.fairseq_tokens_to_ids = {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}\n        self.fairseq_offset = len(self.fairseq_tokens_to_ids)\n        self.fairseq_tokens_to_ids['<mask>'] = len(self.sp_model) + len(self.fairseq_tokens_to_ids)\n        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}\n\n    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):\n        \"\"\"\n        Build model inputs from a sequence or a pair of sequence for sequence classification tasks\n        by concatenating and adding special tokens.\n        A RoBERTa sequence has the following format:\n            single sequence: <s> X </s>\n            pair of sequences: <s> A </s></s> B </s>\n        \"\"\"\n        if token_ids_1 is None:\n            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]\n        cls = [self.cls_token_id]\n        sep = [self.sep_token_id]\n        return cls + token_ids_0 + sep + sep + token_ids_1 + sep\n\n    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):\n        \"\"\"\n        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding\n        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.\n\n        Args:\n            token_ids_0: list of ids (must not contain special tokens)\n            token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids\n                for sequence pairs\n            already_has_special_tokens: (default False) Set to True if the token list is already formated with\n                special tokens for the model\n\n        Returns:\n            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.\n        \"\"\"\n        if already_has_special_tokens:\n            if token_ids_1 is not None:\n                raise ValueError(\"You should not supply a second sequence if the provided sequence of \"\n                                 \"ids is already formated with special tokens for the model.\")\n            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))\n\n        if token_ids_1 is None:\n            return [1] + ([0] * len(token_ids_0)) + [1]\n        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]\n\n    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):\n        \"\"\"\n        Creates a mask from the two sequences passed to be used in a sequence-pair classification task.\n        A RoBERTa sequence pair mask has the following format:\n        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1\n        | first sequence    | second sequence\n\n        if token_ids_1 is None, only returns the first portion of the mask (0's).\n        \"\"\"\n        sep = [self.sep_token_id]\n        cls = [self.cls_token_id]\n\n        if token_ids_1 is None:\n            return len(cls + token_ids_0 + sep) * [0]\n        return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1]\n\n    @property\n    def vocab_size(self):\n        return len(self.fairseq_tokens_to_ids) + len(self.sp_model)\n\n    def _tokenize(self, text):\n        return self.sp_model.EncodeAsPieces(text)\n\n    def _convert_token_to_id(self, token):\n        \"\"\" Converts a token (str/unicode) in an id using the vocab. \"\"\"\n        if token in self.fairseq_tokens_to_ids:\n            return self.fairseq_tokens_to_ids[token]\n        elif self.sp_model.PieceToId(token) == 0:\n            # Convert sentence piece unk token to fairseq unk token index\n            return self.unk_token_id\n        return self.fairseq_offset + self.sp_model.PieceToId(token)\n\n    def _convert_id_to_token(self, index):\n        \"\"\"Converts an index (integer) in a token (string/unicode) using the vocab.\"\"\"\n        if index in self.fairseq_ids_to_tokens:\n            return self.fairseq_ids_to_tokens[index]\n        return self.sp_model.IdToPiece(index - self.fairseq_offset)\n\n    def save_vocabulary(self, save_directory):\n        \"\"\" Save the sentencepiece vocabulary (copy original file) and special tokens file\n            to a directory.\n        \"\"\"\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])\n\n        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):\n            copyfile(self.vocab_file, out_vocab_file)\n\n        return (out_vocab_file,)\n", "path": "transformers/tokenization_camembert.py"}]}