



'''simple docstring'''



import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


lowerCamelCase	 =       1e-4

if is_torch_available():
		import torch

		from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
		from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder





@require_torch
class _UpperCamelCase		:





				'''simple docstring'''





				def __init__(							self  :  Tuple  ,       _lowerCAmelCase  :  List[str]  ,       _lowerCAmelCase  :  Dict=1_6  ,       _lowerCAmelCase  :  Tuple=1_3  ,       _lowerCAmelCase  :  Any=7  ,       _lowerCAmelCase  :  Tuple=1_4  ,       _lowerCAmelCase  :  Optional[Any]=1_0  ,       _lowerCAmelCase  :  int=1_9  ,       _lowerCAmelCase  :  int=5  ,       _lowerCAmelCase  :  Union[str, Any]=4  ,       _lowerCAmelCase  :  Optional[int]=True  ,       _lowerCAmelCase  :  List[Any]=1_6  ,       _lowerCAmelCase  :  Optional[Any]=2  ,       _lowerCAmelCase  :  Optional[int]=4  ,       _lowerCAmelCase  :  Union[str, Any]=4  ,       _lowerCAmelCase  :  List[Any]="gelu"  ,       _lowerCAmelCase  :  Union[str, Any]=0.1  ,       _lowerCAmelCase  :  Any=0.1  ,       _lowerCAmelCase  :  List[str]=[1, 2, 3, 4, 5]  ,       _lowerCAmelCase  :  Tuple=2_5  ,       _lowerCAmelCase  :  List[Any]=5  ,       ):







											'''simple docstring'''

											__lowercase       	=d_model
											__lowercase       	=parent
											__lowercase       	=batch_size
											__lowercase       	=prediction_length
											__lowercase       	=context_length
											__lowercase       	=cardinality
											__lowercase       	=num_time_features
											__lowercase       	=lags_sequence
											__lowercase       	=embedding_dimension
											__lowercase       	=is_training
											__lowercase       	=hidden_size
											__lowercase       	=num_hidden_layers
											__lowercase       	=num_attention_heads
											__lowercase       	=intermediate_size
											__lowercase       	=hidden_act
											__lowercase       	=hidden_dropout_prob
											__lowercase       	=attention_probs_dropout_prob

											__lowercase       	=context_length
											__lowercase       	=prediction_length + label_length
											__lowercase       	=label_length

											__lowercase       	=moving_average
											__lowercase       	=autocorrelation_factor

				def 				__lowerCamelCase     (							self  :  Tuple):







											'''simple docstring'''

											return AutoformerConfig(
											    d_model=self.d_model  ,       encoder_layers=self.num_hidden_layers  ,       decoder_layers=self.num_hidden_layers  ,       encoder_attention_heads=self.num_attention_heads  ,       decoder_attention_heads=self.num_attention_heads  ,       encoder_ffn_dim=self.intermediate_size  ,       decoder_ffn_dim=self.intermediate_size  ,       dropout=self.hidden_dropout_prob  ,       attention_dropout=self.attention_probs_dropout_prob  ,       prediction_length=self.prediction_length  ,       context_length=self.context_length  ,       label_length=self.label_length  ,       lags_sequence=self.lags_sequence  ,       num_time_features=self.num_time_features  ,       num_static_categorical_features=1  ,       cardinality=[self.cardinality]  ,       embedding_dimension=[self.embedding_dimension]  ,       moving_average=self.moving_average  ,       )

				def 				__lowerCamelCase     (							self  :  Optional[Any]  ,       _lowerCAmelCase  :  int):







											'''simple docstring'''

											__lowercase       	=config.context_length + max(config.lags_sequence)

											__lowercase       	=ids_tensor([self.batch_size, 1]  ,       config.cardinality[0])
											__lowercase       	=floats_tensor([self.batch_size, _past_length, config.num_time_features])
											__lowercase       	=floats_tensor([self.batch_size, _past_length])
											__lowercase       	=floats_tensor([self.batch_size, _past_length]) > 0.5

											# decoder inputs
											__lowercase       	=floats_tensor([self.batch_size, config.prediction_length, config.num_time_features])
											__lowercase       	=floats_tensor([self.batch_size, config.prediction_length])

											__lowercase       	={
											    "past_values": past_values,
											    "static_categorical_features": static_categorical_features,
											    "past_time_features": past_time_features,
											    "past_observed_mask": past_observed_mask,
											    "future_time_features": future_time_features,
											    "future_values": future_values,
											}
											return inputs_dict

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											__lowercase       	=self.get_config()
											__lowercase       	=self.prepare_autoformer_inputs_dict(__A)
											return config, inputs_dict

				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											__lowercase       	=self.prepare_config_and_inputs()
											return config, inputs_dict

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Any  ,       _lowerCAmelCase  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=AutoformerModel(config=__A).to(__A).eval()
											__lowercase       	=model(**__A)

											__lowercase       	=outputs.encoder_last_hidden_state
											__lowercase       	=outputs.last_hidden_state

											with tempfile.TemporaryDirectory() as tmpdirname:
																		__lowercase       	=model.get_encoder()
																		encoder.save_pretrained(__A)
																		__lowercase       	=AutoformerEncoder.from_pretrained(__A).to(__A)

											__lowercase       	=model.create_network_inputs(**__A)
											__lowercase       	=model.decomposition_layer(transformer_inputs[:, : config.context_length, ...])

											__lowercase       	=torch.cat(
											    (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])  ,       dim=-1  ,       )
											__lowercase       	=encoder(inputs_embeds=__A)[0]
											self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1e-3)

											__lowercase       	=(
											    torch.mean(transformer_inputs[:, : config.context_length, ...]  ,       dim=1)
											    .unsqueeze(1)
											    .repeat(1  ,       config.prediction_length  ,       1)
											)
											__lowercase       	=torch.zeros(
											    [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]  ,       device=enc_input.device  ,       )

											__lowercase       	=torch.cat(
											    (
											        torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)  ,       dim=1),
											        feature[:, config.context_length - config.label_length :, ...],
											    )  ,       dim=-1  ,       )
											__lowercase       	=torch.cat(
											    (
											        torch.cat((trend_input[:, -config.label_length :, ...], mean)  ,       dim=1),
											        feature[:, config.context_length - config.label_length :, ...],
											    )  ,       dim=-1  ,       )

											with tempfile.TemporaryDirectory() as tmpdirname:
																		__lowercase       	=model.get_decoder()
																		decoder.save_pretrained(__A)
																		__lowercase       	=AutoformerDecoder.from_pretrained(__A).to(__A)

											__lowercase       	=decoder(
											    trend=__A  ,       inputs_embeds=__A  ,       encoder_hidden_states=__A  ,       )[0]

											self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1e-3)





@require_torch
class _UpperCamelCase		(     A      ,       A      ,       unittest.TestCase		):





				'''simple docstring'''





				lowerCAmelCase__					    =    (AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
				lowerCAmelCase__					    =    (AutoformerForPrediction,) if is_torch_available() else ()
				lowerCAmelCase__					    =    {"""feature-extraction""": AutoformerModel} if is_torch_available() else {}
				lowerCAmelCase__					    =    False
				lowerCAmelCase__					    =    False
				lowerCAmelCase__					    =    False
				lowerCAmelCase__					    =    False
				lowerCAmelCase__					    =    False
				lowerCAmelCase__					    =    False

				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											__lowercase       	=AutoformerModelTester(self)
											__lowercase       	=ConfigTester(self  ,       config_class=__A  ,       has_text_modality=__A)

				def 				__lowerCamelCase     (							self  :  int):







											'''simple docstring'''

											self.config_tester.run_common_tests()

				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs()
											for model_class in self.all_model_classes:
																		__lowercase       	=model_class(__A)

																		with tempfile.TemporaryDirectory() as tmpdirname:
																									model.save_pretrained(__A)
																									__lowercase       	=model_class.from_pretrained(__A  ,       output_loading_info=__A)
																		self.assertEqual(info['missing_keys']  ,       [])

				def 				__lowerCamelCase     (							self  :  Tuple):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs_for_common()
											self.model_tester.check_encoder_decoder_model_standalone(*__A)

				@unittest.skip(reason='Model has no tokens embeddings')
				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											pass

				def 				__lowerCamelCase     (							self  :  Tuple):







											'''simple docstring'''

											__lowercase       	=inspect.signature(getattr(__A  ,       'forward'))
											# The main input is the name of the argument after `self`
											__lowercase       	=list(model_signature.parameters.keys())[1]
											self.assertEqual(AutoformerModel.main_input_name  ,       __A)

				def 				__lowerCamelCase     (							self  :  Any):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		__lowercase       	=model_class(__A)
																		__lowercase       	=inspect.signature(model.forward)
																		# signature.parameters is an OrderedDict => so arg_names order is deterministic
																		__lowercase       	=[*signature.parameters.keys()]

																		__lowercase       	=[
																		    "past_values",
																		    "past_time_features",
																		    "past_observed_mask",
																		    "static_categorical_features",
																		    "static_real_features",
																		    "future_values",
																		    "future_time_features",
																		]

																		if model.__class__.__name__ in ["AutoformerForPrediction"]:
																									expected_arg_names.append('future_observed_mask')

																		expected_arg_names.extend(
																		    [
																		        'decoder_attention_mask',
																		        'head_mask',
																		        'decoder_head_mask',
																		        'cross_attn_head_mask',
																		        'encoder_outputs',
																		        'past_key_values',
																		        'output_hidden_states',
																		        'output_attentions',
																		        'use_cache',
																		        'return_dict',
																		    ])

																		self.assertListEqual(arg_names[: len(__A)]  ,       __A)

				def 				__lowerCamelCase     (							self  :  List[Any]):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs_for_common()
											__lowercase       	=True

											__lowercase       	=getattr(self.model_tester  ,       'seq_length'  ,       __A)
											__lowercase       	=getattr(self.model_tester  ,       'decoder_seq_length'  ,       __A)
											__lowercase       	=getattr(self.model_tester  ,       'encoder_seq_length'  ,       __A)
											__lowercase       	=getattr(self.model_tester  ,       'd_model'  ,       __A)
											__lowercase       	=getattr(self.model_tester  ,       'num_attention_heads'  ,       __A)
											__lowercase       	=d_model // num_attention_heads

											for model_class in self.all_model_classes:
																		__lowercase       	=True
																		__lowercase       	=False
																		__lowercase       	=True
																		__lowercase       	=model_class(__A)
																		model.to(__A)
																		model.eval()
																		with torch.no_grad():
																									__lowercase       	=model(**self._prepare_for_class(__A  ,       __A))
																		__lowercase       	=outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
																		self.assertEqual(len(__A)  ,       self.model_tester.num_hidden_layers)

																		# check that output_attentions also work using config
																		del inputs_dict["output_attentions"]
																		__lowercase       	=True
																		__lowercase       	=model_class(__A)
																		model.to(__A)
																		model.eval()
																		with torch.no_grad():
																									__lowercase       	=model(**self._prepare_for_class(__A  ,       __A))
																		__lowercase       	=outputs.encoder_attentions
																		self.assertEqual(len(__A)  ,       self.model_tester.num_hidden_layers)

																		self.assertListEqual(
																		    list(attentions[0].shape[-3:])  ,       [self.model_tester.num_attention_heads, encoder_seq_length, dim]  ,       )
																		__lowercase       	=len(__A)

																		__lowercase       	=7

																		if "last_hidden_state" in outputs:
																									correct_outlen += 1

																		if "trend" in outputs:
																									correct_outlen += 1

																		if "past_key_values" in outputs:
																									correct_outlen += 1  # past_key_values have been returned

																		if "loss" in outputs:
																									correct_outlen += 1

																		if "params" in outputs:
																									correct_outlen += 1

																		self.assertEqual(__A  ,       __A)

																		# decoder attentions
																		__lowercase       	=outputs.decoder_attentions
																		self.assertIsInstance(__A  ,       (list, tuple))
																		self.assertEqual(len(__A)  ,       self.model_tester.num_hidden_layers)
																		self.assertListEqual(
																		    list(decoder_attentions[0].shape[-3:])  ,       [self.model_tester.num_attention_heads, decoder_seq_length, dim]  ,       )

																		# cross attentions
																		__lowercase       	=outputs.cross_attentions
																		self.assertIsInstance(__A  ,       (list, tuple))
																		self.assertEqual(len(__A)  ,       self.model_tester.num_hidden_layers)
																		self.assertListEqual(
																		    list(cross_attentions[0].shape[-3:])  ,       [self.model_tester.num_attention_heads, decoder_seq_length, dim]  ,       )

											# Check attention is always last and order is fine
											__lowercase       	=True
											__lowercase       	=True
											__lowercase       	=model_class(__A)
											model.to(__A)
											model.eval()
											with torch.no_grad():
																		__lowercase       	=model(**self._prepare_for_class(__A  ,       __A))

											self.assertEqual(out_len + 2  ,       len(__A))

											__lowercase       	=outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

											self.assertEqual(len(__A)  ,       self.model_tester.num_hidden_layers)
											self.assertListEqual(
											    list(self_attentions[0].shape[-3:])  ,       [self.model_tester.num_attention_heads, encoder_seq_length, dim]  ,       )

				@is_flaky()
				def 				__lowerCamelCase     (							self  :  Optional[int]):







											'''simple docstring'''

											super().test_retain_grad_hidden_states_attentions()






def   _A       (   _lowerCAmelCase="train-batch.pt"					):



							"""simple docstring"""



							__lowercase       	=hf_hub_download(repo_id='hf-internal-testing/tourism-monthly-batch'  ,   filename=_lowerCAmelCase  ,   repo_type='dataset'					)
							__lowercase       	=torch.load(_lowerCAmelCase  ,   map_location=_lowerCAmelCase					)
							return batch





@require_torch
@slow
class _UpperCamelCase		(     unittest.TestCase		):





				'''simple docstring'''





				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											__lowercase       	=AutoformerModel.from_pretrained('huggingface/autoformer-tourism-monthly').to(__A)
											__lowercase       	=prepare_batch()

											with torch.no_grad():
																		__lowercase       	=model(
																		    past_values=batch['past_values']  ,       past_time_features=batch['past_time_features']  ,       past_observed_mask=batch['past_observed_mask']  ,       static_categorical_features=batch['static_categorical_features']  ,       future_values=batch['future_values']  ,       future_time_features=batch['future_time_features']  ,       )[0]

											__lowercase       	=torch.Size(
											    (6_4, model.config.prediction_length + model.config.label_length, model.config.feature_size))
											self.assertEqual(output.shape  ,       __A)

											__lowercase       	=torch.tensor(
											    [[0.3593, -1.3398, 0.6330], [0.2279, 1.5396, -0.1792], [0.0450, 1.3225, -0.2335]]  ,       device=__A)
											self.assertTrue(torch.allclose(output[0, :3, :3]  ,       __A  ,       atol=__A))

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly').to(__A)
											__lowercase       	=prepare_batch('val-batch.pt')
											with torch.no_grad():
																		__lowercase       	=model(
																		    past_values=batch['past_values']  ,       past_time_features=batch['past_time_features']  ,       past_observed_mask=batch['past_observed_mask']  ,       static_categorical_features=batch['static_categorical_features']  ,       ).encoder_last_hidden_state
											__lowercase       	=torch.Size((6_4, model.config.context_length, model.config.d_model))
											self.assertEqual(output.shape  ,       __A)

											__lowercase       	=torch.tensor(
											    [[-0.0734, -0.9036, 0.8358], [4.7186, 2.4113, 1.9581], [1.7953, 2.3558, 1.2970]]  ,       device=__A)
											self.assertTrue(torch.allclose(output[0, :3, :3]  ,       __A  ,       atol=__A))

				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											__lowercase       	=AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly').to(__A)
											__lowercase       	=prepare_batch('val-batch.pt')
											with torch.no_grad():
																		__lowercase       	=model.generate(
																		    static_categorical_features=batch['static_categorical_features']  ,       past_time_features=batch['past_time_features']  ,       past_values=batch['past_values']  ,       future_time_features=batch['future_time_features']  ,       past_observed_mask=batch['past_observed_mask']  ,       )
											__lowercase       	=torch.Size((6_4, model.config.num_parallel_samples, model.config.prediction_length))
											self.assertEqual(outputs.sequences.shape  ,       __A)

											__lowercase       	=torch.tensor([3130.6763, 4056.5293, 7053.0786]  ,       device=__A)
											__lowercase       	=outputs.sequences.mean(dim=1)
											self.assertTrue(torch.allclose(mean_prediction[0, -3:]  ,       __A  ,       rtol=1e-1))








'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)


lowerCamelCase	 =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
lowerCamelCase	 =       {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

lowerCamelCase	 =       {
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    ["""input_ids""", """attention_mask"""]
				lowerCAmelCase__					    =    BartTokenizer

				def __init__(							self  :  Optional[int]  ,       _lowerCAmelCase  :  Any=None  ,       _lowerCAmelCase  :  Tuple=None  ,       _lowerCAmelCase  :  Optional[int]=None  ,       _lowerCAmelCase  :  str="replace"  ,       _lowerCAmelCase  :  List[Any]="<s>"  ,       _lowerCAmelCase  :  int="</s>"  ,       _lowerCAmelCase  :  Dict="</s>"  ,       _lowerCAmelCase  :  Optional[int]="<s>"  ,       _lowerCAmelCase  :  str="<unk>"  ,       _lowerCAmelCase  :  List[str]="<pad>"  ,       _lowerCAmelCase  :  Dict="<mask>"  ,       _lowerCAmelCase  :  Tuple=False  ,       _lowerCAmelCase  :  str=True  ,       **_lowerCAmelCase  :  Tuple  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       errors=_lowerCAmelCase  ,       bos_token=_lowerCAmelCase  ,       eos_token=_lowerCAmelCase  ,       sep_token=_lowerCAmelCase  ,       cls_token=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       pad_token=_lowerCAmelCase  ,       mask_token=_lowerCAmelCase  ,       add_prefix_space=_lowerCAmelCase  ,       trim_offsets=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											__lowercase       	=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																		__lowercase       	=getattr(_lowerCAmelCase  ,       pre_tok_state.pop('type'))
																		__lowercase       	=add_prefix_space
																		__lowercase       	=pre_tok_class(**_lowerCAmelCase)

											__lowercase       	=add_prefix_space

											# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
											__lowercase       	='post_processor'
											__lowercase       	=getattr(self.backend_tokenizer  ,       _lowerCAmelCase  ,       _lowerCAmelCase)
											if tokenizer_component_instance:
																		__lowercase       	=json.loads(tokenizer_component_instance.__getstate__())

																		# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																		if "sep" in state:
																									__lowercase       	=tuple(state['sep'])
																		if "cls" in state:
																									__lowercase       	=tuple(state['cls'])

																		__lowercase       	=False

																		if state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																									__lowercase       	=add_prefix_space
																									__lowercase       	=True

																		if state.get('trim_offsets'  ,       _lowerCAmelCase) != trim_offsets:
																									__lowercase       	=trim_offsets
																									__lowercase       	=True

																		if changes_to_apply:
																									__lowercase       	=getattr(_lowerCAmelCase  ,       state.pop('type'))
																									__lowercase       	=component_class(**_lowerCAmelCase)
																									setattr(self.backend_tokenizer  ,       _lowerCAmelCase  ,       _lowerCAmelCase)

				@property
				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											if self._mask_token is None:
																		if self.verbose:
																									logger.error('Using mask_token, but it is not set yet.')
																		return None
											return str(self._mask_token)

				@mask_token.setter
				def 				__lowerCamelCase     (							self  :  Union[str, Any]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											__lowercase       	=AddedToken(_lowerCAmelCase  ,       lstrip=_lowerCAmelCase  ,       rstrip=_lowerCAmelCase) if isinstance(_lowerCAmelCase  ,       _lowerCAmelCase) else value
											__lowercase       	=value

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       *_lowerCAmelCase  :  int  ,       **_lowerCAmelCase  :  str):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    'to use it with pretokenized inputs.')

											return super()._batch_encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       *_lowerCAmelCase  :  List[str]  ,       **_lowerCAmelCase  :  Tuple):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    'to use it with pretokenized inputs.')

											return super()._encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  int  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  List[str]=None):







											'''simple docstring'''

											__lowercase       	=[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																		return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

				def 				__lowerCamelCase     (							self  :  List[str]  ,       _lowerCAmelCase  :  List[int]  ,       _lowerCAmelCase  :  Optional[List[int]] = None):







											'''simple docstring'''

											__lowercase       	=[self.sep_token_id]
											__lowercase       	=[self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



