






'''simple docstring'''







import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    MegatronBertForCausalLM,
							    MegatronBertForMaskedLM,
							    MegatronBertForMultipleChoice,
							    MegatronBertForNextSentencePrediction,
							    MegatronBertForPreTraining,
							    MegatronBertForQuestionAnswering,
							    MegatronBertForSequenceClassification,
							    MegatronBertForTokenClassification,
							    MegatronBertModel,
							)




class 		UpperCamelCase__    :




		'''simple docstring'''





		def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase=13   ,_lowerCAmelCase=7   ,_lowerCAmelCase=True   ,_lowerCAmelCase=True   ,_lowerCAmelCase=True   ,_lowerCAmelCase=True   ,_lowerCAmelCase=99   ,_lowerCAmelCase=64   ,_lowerCAmelCase=32   ,_lowerCAmelCase=5   ,_lowerCAmelCase=4   ,_lowerCAmelCase=37   ,_lowerCAmelCase="gelu"   ,_lowerCAmelCase=0.1   ,_lowerCAmelCase=0.1   ,_lowerCAmelCase=5_12   ,_lowerCAmelCase=16   ,_lowerCAmelCase=2   ,_lowerCAmelCase=0.02   ,_lowerCAmelCase=3   ,_lowerCAmelCase=4   ,_lowerCAmelCase=None   ,):
								lowerCamelCase__         =  parent
								lowerCamelCase__         =  batch_size
								lowerCamelCase__         =  seq_length
								lowerCamelCase__         =  is_training
								lowerCamelCase__         =  use_input_mask
								lowerCamelCase__         =  use_token_type_ids
								lowerCamelCase__         =  use_labels
								lowerCamelCase__         =  vocab_size
								lowerCamelCase__         =  hidden_size
								lowerCamelCase__         =  embedding_size
								lowerCamelCase__         =  num_hidden_layers
								lowerCamelCase__         =  num_attention_heads
								lowerCamelCase__         =  intermediate_size
								lowerCamelCase__         =  hidden_act
								lowerCamelCase__         =  hidden_dropout_prob
								lowerCamelCase__         =  attention_probs_dropout_prob
								lowerCamelCase__         =  max_position_embeddings
								lowerCamelCase__         =  type_vocab_size
								lowerCamelCase__         =  type_sequence_label_size
								lowerCamelCase__         =  initializer_range
								lowerCamelCase__         =  num_labels
								lowerCamelCase__         =  num_choices
								lowerCamelCase__         =  scope






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  ids_tensor([self.batch_size, self.seq_length]   ,self.vocab_size       )

								lowerCamelCase__         =  None
								if self.use_input_mask:
														lowerCamelCase__         =  random_attention_mask([self.batch_size, self.seq_length]       )

								lowerCamelCase__         =  None
								if self.use_token_type_ids:
														lowerCamelCase__         =  ids_tensor([self.batch_size, self.seq_length]   ,self.type_vocab_size       )

								lowerCamelCase__         =  None
								lowerCamelCase__         =  None
								lowerCamelCase__         =  None
								if self.use_labels:
														lowerCamelCase__         =  ids_tensor([self.batch_size]   ,self.type_sequence_label_size       )
														lowerCamelCase__         =  ids_tensor([self.batch_size, self.seq_length]   ,self.num_labels       )
														lowerCamelCase__         =  ids_tensor([self.batch_size]   ,self.num_choices       )

								lowerCamelCase__         =  self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






		def 			UpperCamelCase_					(      self       ):
								return MegatronBertConfig(
								    vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,intermediate_size=self.intermediate_size   ,embedding_size=self.embedding_size   ,hidden_act=self.hidden_act   ,hidden_dropout_prob=self.hidden_dropout_prob   ,attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,type_vocab_size=self.type_vocab_size   ,is_decoder=_lowerCAmelCase   ,initializer_range=self.initializer_range   ,)






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  MegatronBertModel(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(_lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase       )
								lowerCamelCase__         =  model(_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase       )
								lowerCamelCase__         =  model(_lowerCAmelCase       )

								self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size)       )
								self.parent.assertEqual(result.pooler_output.shape   ,(self.batch_size, self.hidden_size)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  MegatronBertForMaskedLM(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(_lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,labels=_lowerCAmelCase       )
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  MegatronBertForCausalLM(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(_lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,labels=_lowerCAmelCase       )
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  MegatronBertForNextSentencePrediction(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(
								    _lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,labels=_lowerCAmelCase   ,)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, 2)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  MegatronBertForPreTraining(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(
								    _lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,labels=_lowerCAmelCase   ,next_sentence_label=_lowerCAmelCase   ,)
								self.parent.assertEqual(result.prediction_logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)       )
								self.parent.assertEqual(result.seq_relationship_logits.shape   ,(self.batch_size, 2)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  MegatronBertForQuestionAnswering(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(
								    _lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,start_positions=_lowerCAmelCase   ,end_positions=_lowerCAmelCase   ,)
								self.parent.assertEqual(result.start_logits.shape   ,(self.batch_size, self.seq_length)       )
								self.parent.assertEqual(result.end_logits.shape   ,(self.batch_size, self.seq_length)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  self.num_labels
								lowerCamelCase__         =  MegatronBertForSequenceClassification(_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(_lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,labels=_lowerCAmelCase       )
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_labels)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  self.num_labels
								lowerCamelCase__         =  MegatronBertForTokenClassification(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  model(_lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,labels=_lowerCAmelCase       )
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.num_labels)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  self.num_choices
								lowerCamelCase__         =  MegatronBertForMultipleChoice(config=_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.eval()
								lowerCamelCase__         =  input_ids.unsqueeze(1       ).expand(-1   ,self.num_choices   ,-1       ).contiguous()
								lowerCamelCase__         =  token_type_ids.unsqueeze(1       ).expand(-1   ,self.num_choices   ,-1       ).contiguous()
								lowerCamelCase__         =  input_mask.unsqueeze(1       ).expand(-1   ,self.num_choices   ,-1       ).contiguous()
								lowerCamelCase__         =  model(
								    _lowerCAmelCase   ,attention_mask=_lowerCAmelCase   ,token_type_ids=_lowerCAmelCase   ,labels=_lowerCAmelCase   ,)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_choices)       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.prepare_config_and_inputs()
								(
								    (
								    lowerCamelCase__
								)       ,			(
								    lowerCamelCase__
								)       ,			(
								    lowerCamelCase__
								)       ,			(
								    lowerCamelCase__
								)       ,			(
								    lowerCamelCase__
								)       ,			(
								    lowerCamelCase__
								)       ,			(
								    lowerCamelCase__
								)       ,			
								)         =  config_and_inputs
								lowerCamelCase__         =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
								return config, inputs_dict




@require_torch
class 		UpperCamelCase__    (a   ,a   ,unittest.TestCase			):




		'''simple docstring'''





		_UpperCamelCase				  =					(
		    (
		        MegatronBertModel,
		        MegatronBertForMaskedLM,
		        MegatronBertForCausalLM,
		        MegatronBertForMultipleChoice,
		        MegatronBertForNextSentencePrediction,
		        MegatronBertForPreTraining,
		        MegatronBertForQuestionAnswering,
		        MegatronBertForSequenceClassification,
		        MegatronBertForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		_UpperCamelCase				  =					(
		    {
		        'feature-extraction': MegatronBertModel,
		        'fill-mask': MegatronBertForMaskedLM,
		        'question-answering': MegatronBertForQuestionAnswering,
		        'text-classification': MegatronBertForSequenceClassification,
		        'text-generation': MegatronBertForCausalLM,
		        'token-classification': MegatronBertForTokenClassification,
		        'zero-shot': MegatronBertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		_UpperCamelCase				  =					True
		# test_resize_embeddings = False
		_UpperCamelCase				  =					False






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase=False       ):
								lowerCamelCase__         =  super()._prepare_for_class(_lowerCAmelCase   ,_lowerCAmelCase   ,return_labels=_lowerCAmelCase       )

								if return_labels:
														if model_class in get_values(_lowerCAmelCase       ):
																				lowerCamelCase__         =  torch.zeros(
																				    (self.model_tester.batch_size, self.model_tester.seq_length)   ,dtype=torch.long   ,device=_lowerCAmelCase       )
																				lowerCamelCase__         =  torch.zeros(
																				    self.model_tester.batch_size   ,dtype=torch.long   ,device=_lowerCAmelCase       )
								return inputs_dict






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  MegatronBertModelTester(self       )
								lowerCamelCase__         =  ConfigTester(self   ,config_class=_lowerCAmelCase   ,hidden_size=37       )






		def 			UpperCamelCase_					(      self       ):
								self.config_tester.run_common_tests()






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_model(*_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_masked_lm(*_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_pretraining(*_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_question_answering(*_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_token_classification(*_lowerCAmelCase       )



def 						A__			(				__lowerCAmelCase		:			Optional[int]						):
						return torch.tensor(
						    __lowerCAmelCase   ,    dtype=torch.long   ,    device=__lowerCAmelCase   ,    )


UpperCamelCase    :      Dict  			=		1E-4





@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__    (unittest.TestCase			):




		'''simple docstring'''





		@slow
		@unittest.skip("""Model is not available."""       )
		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  """nvidia/megatron-bert-uncased-345m"""
								if "MYDIR" in os.environ:
														lowerCamelCase__         =  os.path.join(os.environ["""MYDIR"""]   ,_lowerCAmelCase       )
								lowerCamelCase__         =  MegatronBertModel.from_pretrained(_lowerCAmelCase       )
								model.to(_lowerCAmelCase       )
								model.half()
								lowerCamelCase__         =  _long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]       )
								with torch.no_grad():
														lowerCamelCase__         =  model(_lowerCAmelCase       )[0]
								lowerCamelCase__         =  torch.Size((1, 9, 10_24)       )
								self.assertEqual(output.shape   ,_lowerCAmelCase       )

								lowerCamelCase__         =  [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
								for ii in range(3       ):
														for jj in range(3       ):
																				lowerCamelCase__         =  output[0, ii, jj]
																				lowerCamelCase__         =  expected[3 * ii + jj]
																				lowerCamelCase__         =  """ii={} jj={} a={} b={}""".format(_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       )
																				self.assertTrue(math.isclose(_lowerCAmelCase   ,_lowerCAmelCase   ,rel_tol=_lowerCAmelCase   ,abs_tol=_lowerCAmelCase       )   ,msg=_lowerCAmelCase       )









'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase    :      int  			=		logging.get_logger(__name__)

UpperCamelCase    :      Optional[int]  			=		{'vocab_file': 'sentencepiece.bpe.model'}

UpperCamelCase    :      Dict  			=		{
    'vocab_file': {
        'camembert-base': 'https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model',
    }
}

UpperCamelCase    :      List[Any]  			=		{
    'camembert-base': 5_12,
}

UpperCamelCase    :      List[str]  			=		'▁'




class 		UpperCamelCase__    (a			):




		'''simple docstring'''





		_UpperCamelCase				  =					VOCAB_FILES_NAMES
		_UpperCamelCase				  =					PRETRAINED_VOCAB_FILES_MAP
		_UpperCamelCase				  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_UpperCamelCase				  =					['input_ids', 'attention_mask']






		def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase="<s>"   ,_lowerCAmelCase="</s>"   ,_lowerCAmelCase="</s>"   ,_lowerCAmelCase="<s>"   ,_lowerCAmelCase="<unk>"   ,_lowerCAmelCase="<pad>"   ,_lowerCAmelCase="<mask>"   ,_lowerCAmelCase=["<s>NOTUSED", "</s>NOTUSED"]   ,_lowerCAmelCase = None   ,**_lowerCAmelCase   ,):
								# Mask token behave like a normal word, i.e. include the space before it
								lowerCamelCase__         =  AddedToken(_lowerCAmelCase   ,lstrip=_lowerCAmelCase   ,rstrip=_lowerCAmelCase       ) if isinstance(_lowerCAmelCase   ,_lowerCAmelCase       ) else mask_token

								lowerCamelCase__         =  {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    bos_token=_lowerCAmelCase   ,eos_token=_lowerCAmelCase   ,unk_token=_lowerCAmelCase   ,sep_token=_lowerCAmelCase   ,cls_token=_lowerCAmelCase   ,pad_token=_lowerCAmelCase   ,mask_token=_lowerCAmelCase   ,additional_special_tokens=_lowerCAmelCase   ,sp_model_kwargs=self.sp_model_kwargs   ,**_lowerCAmelCase   ,)
								lowerCamelCase__         =  spm.SentencePieceProcessor(**self.sp_model_kwargs       )
								self.sp_model.Load(str(_lowerCAmelCase       )       )
								lowerCamelCase__         =  vocab_file
								# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
								# sentencepiece vocabulary (this is the case for <s> and </s>
								lowerCamelCase__         =  {"""<s>NOTUSED""": 0, """<pad>""": 1, """</s>NOTUSED""": 2, """<unk>""": 3}
								lowerCamelCase__         =  len(self.fairseq_tokens_to_ids       )
								lowerCamelCase__         =  len(self.sp_model       ) + len(self.fairseq_tokens_to_ids       )
								lowerCamelCase__         =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase = None       ):

								if token_ids_a is None:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowerCamelCase__         =  [self.cls_token_id]
								lowerCamelCase__         =  [self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase = None   ,_lowerCAmelCase = False       ):
								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_lowerCAmelCase   ,token_ids_a=_lowerCAmelCase   ,already_has_special_tokens=_lowerCAmelCase       )

								if token_ids_a is None:
														return [1] + ([0] * len(_lowerCAmelCase       )) + [1]
								return [1] + ([0] * len(_lowerCAmelCase       )) + [1, 1] + ([0] * len(_lowerCAmelCase       )) + [1]






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase = None       ):
								lowerCamelCase__         =  [self.sep_token_id]
								lowerCamelCase__         =  [self.cls_token_id]

								if token_ids_a is None:
														return len(cls + token_ids_a + sep       ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]






		@property
		def 			UpperCamelCase_					(      self       ):
								return len(self.fairseq_tokens_to_ids       ) + len(self.sp_model       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  {self.convert_ids_to_tokens(_lowerCAmelCase       ): i for i in range(self.vocab_size       )}
								vocab.update(self.added_tokens_encoder       )
								return vocab






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								return self.sp_model.encode(_lowerCAmelCase   ,out_type=_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								if token in self.fairseq_tokens_to_ids:
														return self.fairseq_tokens_to_ids[token]
								elif self.sp_model.PieceToId(_lowerCAmelCase       ) == 0:
														# Convert sentence piece unk token to fairseq unk token index
														return self.unk_token_id
								return self.fairseq_offset + self.sp_model.PieceToId(_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								if index in self.fairseq_ids_to_tokens:
														return self.fairseq_ids_to_tokens[index]
								return self.sp_model.IdToPiece(index - self.fairseq_offset       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  []
								lowerCamelCase__         =  """"""
								lowerCamelCase__         =  False
								for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																				if not prev_is_special:
																										out_string += " "
																				out_string += self.sp_model.decode(_lowerCAmelCase       ) + token
																				lowerCamelCase__         =  True
																				lowerCamelCase__         =  []
														else:
																				current_sub_tokens.append(_lowerCAmelCase       )
																				lowerCamelCase__         =  False
								out_string += self.sp_model.decode(_lowerCAmelCase       )
								return out_string.strip()






		def __getstate__(      self       ):
								lowerCamelCase__         =  self.__dict__.copy()
								lowerCamelCase__         =  None
								return state






		def __setstate__(      self   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  d

								# for backward compatibility
								if not hasattr(self   ,"""sp_model_kwargs"""       ):
														lowerCamelCase__         =  {}

								lowerCamelCase__         =  spm.SentencePieceProcessor(**self.sp_model_kwargs       )
								self.sp_model.Load(self.vocab_file       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase = None       ):
								if not os.path.isdir(_lowerCAmelCase       ):
														logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
														return
								lowerCamelCase__         =  os.path.join(
								    _lowerCAmelCase   ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

								if os.path.abspath(self.vocab_file       ) != os.path.abspath(_lowerCAmelCase       ) and os.path.isfile(self.vocab_file       ):
														copyfile(self.vocab_file   ,_lowerCAmelCase       )
								elif not os.path.isfile(self.vocab_file       ):
														with open(_lowerCAmelCase   ,"""wb"""       ) as fi:
																				lowerCamelCase__         =  self.sp_model.serialized_model_proto()
																				fi.write(_lowerCAmelCase       )

								return (out_vocab_file,)

