


"""simple docstring"""



import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
     CTRLForSequenceClassification,
     CTRLLMHeadModel,
     CTRLModel,
 )
class 					__magic_name__		:






       def __init__(   self  :    Optional[Any]  ,     snake_case__  :    List[str]  ,     snake_case__  :    Optional[Any]=1_4  ,     snake_case__  :    Union[str, Any]=7  ,     snake_case__  :    Union[str, Any]=True  ,     snake_case__  :    List[Any]=True  ,     snake_case__  :    Union[str, Any]=True  ,     snake_case__  :    Any=True  ,     snake_case__  :    Any=True  ,     snake_case__  :    int=9_9  ,     snake_case__  :    List[str]=3_2  ,     snake_case__  :    Any=5  ,     snake_case__  :    List[str]=4  ,     snake_case__  :    Union[str, Any]=3_7  ,     snake_case__  :    Dict="gelu"  ,     snake_case__  :    List[Any]=0.1  ,     snake_case__  :    Union[str, Any]=0.1  ,     snake_case__  :    int=5_1_2  ,     snake_case__  :    Optional[Any]=1_6  ,     snake_case__  :    Any=2  ,     snake_case__  :    str=0.02  ,     snake_case__  :    Any=3  ,     snake_case__  :    Optional[int]=4  ,     snake_case__  :    Tuple=None  ,     ):

              '''simple docstring'''






              lowercase		:List[Any]       							=				parent
              lowercase		:Tuple       							=				batch_size
              lowercase		:Optional[int]       							=				seq_length
              lowercase		:Tuple       							=				is_training
              lowercase		:Union[str, Any]       							=				use_token_type_ids
              lowercase		:Dict       							=				use_input_mask
              lowercase		:Optional[Any]       							=				use_labels
              lowercase		:List[str]       							=				use_mc_token_ids
              lowercase		:Dict       							=				vocab_size
              lowercase		:List[str]       							=				hidden_size
              lowercase		:int       							=				num_hidden_layers
              lowercase		:Tuple       							=				num_attention_heads
              lowercase		:List[str]       							=				intermediate_size
              lowercase		:str       							=				hidden_act
              lowercase		:Any       							=				hidden_dropout_prob
              lowercase		:Optional[Any]       							=				attention_probs_dropout_prob
              lowercase		:Tuple       							=				max_position_embeddings
              lowercase		:List[str]       							=				type_vocab_size
              lowercase		:List[str]       							=				type_sequence_label_size
              lowercase		:Any       							=				initializer_range
              lowercase		:str       							=				num_labels
              lowercase		:Optional[int]       							=				num_choices
              lowercase		:int       							=				scope
              lowercase		:Any       							=				self.vocab_size - 1






       def  __snake_case						(   self  :    Any				):

              '''simple docstring'''






              lowercase		:str       							=				ids_tensor([self.batch_size, self.seq_length]  ,     self.vocab_size				)

              lowercase		:Dict       							=				None
              if self.use_input_mask:
                     lowercase		:str       							=				random_attention_mask([self.batch_size, self.seq_length]				)

              lowercase		:str       							=				None
              if self.use_token_type_ids:
                     lowercase		:Tuple       							=				ids_tensor([self.batch_size, self.seq_length]  ,     self.type_vocab_size				)

              lowercase		:Dict       							=				None
              if self.use_mc_token_ids:
                     lowercase		:Optional[Any]       							=				ids_tensor([self.batch_size, self.num_choices]  ,     self.seq_length				)

              lowercase		:Optional[Any]       							=				None
              lowercase		:Optional[int]       							=				None
              lowercase		:List[str]       							=				None
              if self.use_labels:
                     lowercase		:int       							=				ids_tensor([self.batch_size]  ,     self.type_sequence_label_size				)
                     lowercase		:Dict       							=				ids_tensor([self.batch_size, self.seq_length]  ,     self.num_labels				)
                     lowercase		:List[str]       							=				ids_tensor([self.batch_size]  ,     self.num_choices				)

              lowercase		:List[Any]       							=				self.get_config()

              lowercase		:int       							=				ids_tensor([self.num_hidden_layers, self.num_attention_heads]  ,     2				)

              return (
                  config,
                  input_ids,
                  input_mask,
                  head_mask,
                  token_type_ids,
                  mc_token_ids,
                  sequence_labels,
                  token_labels,
                  choice_labels,
              )






       def  __snake_case						(   self  :    str				):

              '''simple docstring'''






              return CTRLConfig(
                  vocab_size=self.vocab_size  ,     n_embd=self.hidden_size  ,     n_layer=self.num_hidden_layers  ,     n_head=self.num_attention_heads  ,     n_positions=self.max_position_embeddings  ,     pad_token_id=self.pad_token_id  ,     )






       def  __snake_case						(   self  :    List[Any]  ,     snake_case__  :    Optional[int]  ,     snake_case__  :    Optional[Any]  ,     snake_case__  :    Tuple  ,     snake_case__  :    List[str]  ,     snake_case__  :    Any  ,     *snake_case__  :    Optional[int]				):

              '''simple docstring'''






              lowercase		:List[str]       							=				CTRLModel(config=snake_case__				)
              model.to(snake_case__				)
              model.eval()

              model(snake_case__  ,     token_type_ids=snake_case__  ,     head_mask=snake_case__				)
              model(snake_case__  ,     token_type_ids=snake_case__				)
              lowercase		:Optional[Any]       							=				model(snake_case__				)
              self.parent.assertEqual(result.last_hidden_state.shape  ,     (self.batch_size, self.seq_length, self.hidden_size)				)
              self.parent.assertEqual(len(result.past_key_values				)  ,     config.n_layer				)






       def  __snake_case						(   self  :    str  ,     snake_case__  :    Optional[int]  ,     snake_case__  :    Any  ,     snake_case__  :    Optional[int]  ,     snake_case__  :    Tuple  ,     snake_case__  :    List[str]  ,     *snake_case__  :    Any				):

              '''simple docstring'''






              lowercase		:Tuple       							=				CTRLLMHeadModel(snake_case__				)
              model.to(snake_case__				)
              model.eval()

              lowercase		:str       							=				model(snake_case__  ,     token_type_ids=snake_case__  ,     labels=snake_case__				)
              self.parent.assertEqual(result.loss.shape  ,     ()				)
              self.parent.assertEqual(result.logits.shape  ,     (self.batch_size, self.seq_length, self.vocab_size)				)






       def  __snake_case						(   self  :    Any				):

              '''simple docstring'''






              lowercase		:Tuple       							=				self.prepare_config_and_inputs()

              (
                  (
                  lowercase
              )							,  (
                  lowercase
              )							,  (
                  lowercase
              )							,  (
                  lowercase
              )							,  (
                  lowercase
              )							,  (
                  lowercase
              )							,  (
                  lowercase
              )							,  (
                  lowercase
              )							,  (
                  lowercase
              )							,  
              )		:List[str]       							=				config_and_inputs

              lowercase		:Any       							=				{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''head_mask''': head_mask}

              return config, inputs_dict







       def  __snake_case						(   self  :    Optional[int]  ,     snake_case__  :    Union[str, Any]  ,     snake_case__  :    Dict  ,     snake_case__  :    Union[str, Any]  ,     snake_case__  :    str  ,     *snake_case__  :    int				):

              '''simple docstring'''






              lowercase		:List[Any]       							=				self.num_labels
              lowercase		:Dict       							=				CTRLForSequenceClassification(snake_case__				)
              model.to(snake_case__				)
              model.eval()
              lowercase		:List[str]       							=				ids_tensor([self.batch_size]  ,     self.type_sequence_label_size				)
              lowercase		:Any       							=				model(snake_case__  ,     token_type_ids=snake_case__  ,     labels=snake_case__				)
              self.parent.assertEqual(result.logits.shape  ,     (self.batch_size, self.num_labels)				)







@require_torch
class 					__magic_name__		(			__UpperCAmelCase			,						__UpperCAmelCase			,						__UpperCAmelCase			,						unittest.TestCase     ):
       __A			:      Union[str, Any]        =   (CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
       __A			:      Tuple        =   (CTRLLMHeadModel,) if is_torch_available() else ()
       __A			:      str        =   (
           {
               "feature-extraction": CTRLModel,
               "text-classification": CTRLForSequenceClassification,
               "text-generation": CTRLLMHeadModel,
               "zero-shot": CTRLForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       __A			:      Union[str, Any]        =   True
       __A			:      Dict        =   False
       __A			:      Tuple        =   False






       def  __snake_case						(   self  :    List[Any]  ,     snake_case__  :    Any  ,     snake_case__  :    List[Any]  ,     snake_case__  :    str  ,     snake_case__  :    str  ,     snake_case__  :    List[str]				):

              '''simple docstring'''






              if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
                     # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
                     # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
                     # config could not be created.
                     return True

              return False






       def  __snake_case						(   self  :    List[str]				):

              '''simple docstring'''






              lowercase		:List[str]       							=				CTRLModelTester(self				)
              lowercase		:Union[str, Any]       							=				ConfigTester(self  ,     config_class=snake_case__  ,     n_embd=3_7				)






       def  __snake_case						(   self  :    int				):

              '''simple docstring'''






              super().tearDown()
              # clean-up as much as possible GPU memory occupied by PyTorch
              gc.collect()
              torch.cuda.empty_cache()






       def  __snake_case						(   self  :    Optional[int]				):

              '''simple docstring'''






              self.config_tester.run_common_tests()






       def  __snake_case						(   self  :    str				):

              '''simple docstring'''






              lowercase		:Union[str, Any]       							=				self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_ctrl_model(*snake_case__				)






       def  __snake_case						(   self  :    List[str]				):

              '''simple docstring'''






              lowercase		:List[Any]       							=				self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_lm_head_model(*snake_case__				)






       @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''				)
       def  __snake_case						(   self  :    List[Any]				):

              '''simple docstring'''






              pass






       @slow
       def  __snake_case						(   self  :    str				):

              '''simple docstring'''






              for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                     lowercase		:Dict       							=				CTRLModel.from_pretrained(snake_case__				)
                     self.assertIsNotNone(snake_case__				)







       @unittest.skip('''The model doesn\'t support left padding'''				)  # and it's not used enough to be worth fixing :)
       def  __snake_case						(   self  :    List[str]				):

              '''simple docstring'''






              pass







@require_torch
class 					__magic_name__		(			unittest.TestCase     ):






       def  __snake_case						(   self  :    Union[str, Any]				):

              '''simple docstring'''






              super().tearDown()
              # clean-up as much as possible GPU memory occupied by PyTorch
              gc.collect()
              torch.cuda.empty_cache()







       @slow
       def  __snake_case						(   self  :    Any				):

              '''simple docstring'''






              lowercase		:Optional[int]       							=				CTRLLMHeadModel.from_pretrained('''ctrl'''				)
              model.to(snake_case__				)
              lowercase		:Union[str, Any]       							=				torch.tensor(
                  [[1_1_8_5_9, 0, 1_6_1_1, 8]]  ,     dtype=torch.long  ,     device=snake_case__				)  # Legal the president is
              lowercase		:List[str]       							=				[
                  1_1_8_5_9,
                  0,
                  1_6_1_1,
                  8,
                  5,
                  1_5_0,
                  2_6_4_4_9,
                  2,
                  1_9,
                  3_4_8,
                  4_6_9,
                  3,
                  2_5_9_5,
                  4_8,
                  2_0_7_4_0,
                  2_4_6_5_3_3,
                  2_4_6_5_3_3,
                  1_9,
                  3_0,
                  5,
              ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

              lowercase		:Any       							=				model.generate(snake_case__  ,     do_sample=snake_case__				)
              self.assertListEqual(output_ids[0].tolist()  ,     snake_case__				)





"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


UpperCAmelCase      		=			logging.get_logger(__name__)


UpperCAmelCase      		=			{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCAmelCase      		=			{
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json''',
    },
}

UpperCAmelCase      		=			{
    '''facebook/bart-base''': 1_024,
    '''facebook/bart-large''': 1_024,
    '''facebook/bart-large-mnli''': 1_024,
    '''facebook/bart-large-cnn''': 1_024,
    '''facebook/bart-large-xsum''': 1_024,
    '''yjernite/bart_eli5''': 1_024,
}
class 					__magic_name__		(			__UpperCAmelCase     ):
       __A			:      str        =   VOCAB_FILES_NAMES
       __A			:      int        =   PRETRAINED_VOCAB_FILES_MAP
       __A			:      List[Any]        =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __A			:      Optional[int]        =   ["input_ids", "attention_mask"]
       __A			:      List[Any]        =   BartTokenizer






       def __init__(   self  :    List[Any]  ,     snake_case__  :    Dict=None  ,     snake_case__  :    Dict=None  ,     snake_case__  :    Dict=None  ,     snake_case__  :    Optional[int]="replace"  ,     snake_case__  :    int="<s>"  ,     snake_case__  :    Optional[Any]="</s>"  ,     snake_case__  :    str="</s>"  ,     snake_case__  :    Union[str, Any]="<s>"  ,     snake_case__  :    Optional[Any]="<unk>"  ,     snake_case__  :    Tuple="<pad>"  ,     snake_case__  :    str="<mask>"  ,     snake_case__  :    List[Any]=False  ,     snake_case__  :    Tuple=True  ,     **snake_case__  :    Dict  ,     ):

              '''simple docstring'''






              super().__init__(
                  snake_case__  ,     snake_case__  ,     tokenizer_file=snake_case__  ,     errors=snake_case__  ,     bos_token=snake_case__  ,     eos_token=snake_case__  ,     sep_token=snake_case__  ,     cls_token=snake_case__  ,     unk_token=snake_case__  ,     pad_token=snake_case__  ,     mask_token=snake_case__  ,     add_prefix_space=snake_case__  ,     trim_offsets=snake_case__  ,     **snake_case__  ,     )

              lowercase		:Dict       							=				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
              if pre_tok_state.get('''add_prefix_space'''  ,     snake_case__				) != add_prefix_space:
                     lowercase		:Union[str, Any]       							=				getattr(snake_case__  ,     pre_tok_state.pop('''type'''				)				)
                     lowercase		:Tuple       							=				add_prefix_space
                     lowercase		:int       							=				pre_tok_class(**snake_case__				)

              lowercase		:str       							=				add_prefix_space

              # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
              lowercase		:Dict       							=				'''post_processor'''
              lowercase		:Optional[int]       							=				getattr(self.backend_tokenizer  ,     snake_case__  ,     snake_case__				)
              if tokenizer_component_instance:
                     lowercase		:Tuple       							=				json.loads(tokenizer_component_instance.__getstate__()				)

                     # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
                     if "sep" in state:
                            lowercase		:Optional[Any]       							=				tuple(state['''sep''']				)
                     if "cls" in state:
                            lowercase		:Tuple       							=				tuple(state['''cls''']				)

                     lowercase		:int       							=				False

                     if state.get('''add_prefix_space'''  ,     snake_case__				) != add_prefix_space:
                            lowercase		:Dict       							=				add_prefix_space
                            lowercase		:Optional[int]       							=				True

                     if state.get('''trim_offsets'''  ,     snake_case__				) != trim_offsets:
                            lowercase		:Any       							=				trim_offsets
                            lowercase		:Dict       							=				True

                     if changes_to_apply:
                            lowercase		:Dict       							=				getattr(snake_case__  ,     state.pop('''type'''				)				)
                            lowercase		:List[str]       							=				component_class(**snake_case__				)
                            setattr(self.backend_tokenizer  ,     snake_case__  ,     snake_case__				)






       @property
       def  __snake_case						(   self  :    Dict				):

              '''simple docstring'''






              if self._mask_token is None:
                     if self.verbose:
                            logger.error('''Using mask_token, but it is not set yet.'''				)
                     return None
              return str(self._mask_token				)






       @mask_token.setter
       def  __snake_case						(   self  :    List[Any]  ,     snake_case__  :    Optional[int]				):

              '''simple docstring'''






              lowercase		:Dict       							=				AddedToken(snake_case__  ,     lstrip=snake_case__  ,     rstrip=snake_case__				) if isinstance(snake_case__  ,     snake_case__				) else value
              lowercase		:Dict       							=				value






       def  __snake_case						(   self  :    Optional[int]  ,     *snake_case__  :    str  ,     **snake_case__  :    str				):

              '''simple docstring'''






              lowercase		:Any       							=				kwargs.get('''is_split_into_words'''  ,     snake_case__				)

              if is_split_into_words and not self.add_prefix_space:
                     raise ValueError(
                         f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                         '''to use it with pretokenized inputs.'''				)

              return super()._batch_encode_plus(*snake_case__  ,     **snake_case__				)






       def  __snake_case						(   self  :    int  ,     *snake_case__  :    int  ,     **snake_case__  :    int				):

              '''simple docstring'''






              lowercase		:Optional[int]       							=				kwargs.get('''is_split_into_words'''  ,     snake_case__				)

              if is_split_into_words and not self.add_prefix_space:
                     raise ValueError(
                         f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                         '''to use it with pretokenized inputs.'''				)

              return super()._encode_plus(*snake_case__  ,     **snake_case__				)






       def  __snake_case						(   self  :    Dict  ,     snake_case__  :    str  ,     snake_case__  :    Optional[str] = None				):

              '''simple docstring'''






              lowercase		:List[Any]       							=				self._tokenizer.model.save(snake_case__  ,     name=snake_case__				)
              return tuple(snake_case__				)






       def  __snake_case						(   self  :    int  ,     snake_case__  :    Any  ,     snake_case__  :    int=None				):

              '''simple docstring'''






              lowercase		:Any       							=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
              if token_ids_a is None:
                     return output

              return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







       def  __snake_case						(   self  :    List[Any]  ,     snake_case__  :    List[int]  ,     snake_case__  :    Optional[List[int]] = None				):

              '''simple docstring'''






              lowercase		:int       							=				[self.sep_token_id]
              lowercase		:Tuple       							=				[self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep				) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]

