'''simple docstring'''




import unittest

import numpy as np

from transformers import RobertaConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
 from transformers.models.roberta.modeling_flax_roberta import (
     FlaxRobertaForCausalLM,
     FlaxRobertaForMaskedLM,
     FlaxRobertaForMultipleChoice,
     FlaxRobertaForQuestionAnswering,
     FlaxRobertaForSequenceClassification,
     FlaxRobertaForTokenClassification,
     FlaxRobertaModel,
 )


class       UpperCAmelCase__  (       unittest.TestCase			):






   """simple docstring"""
   def __init__(self       ,				_a       ,				_a=13       ,				_a=7       ,				_a=True       ,				_a=True       ,				_a=True       ,				_a=True       ,				_a=99       ,				_a=32       ,				_a=5       ,				_a=4       ,				_a=37       ,				_a="gelu"       ,				_a=0.1       ,				_a=0.1       ,				_a=512       ,				_a=16       ,				_a=2       ,				_a=0.02       ,				_a=4       ,				)    ->     Any:
        lowercase_					:       Optional[int]     				=					parent
        lowercase_					:       Optional[Any]     				=					batch_size
        lowercase_					:       Tuple     				=					seq_length
        lowercase_					:       Any     				=					is_training
        lowercase_					:       Optional[int]     				=					use_attention_mask
        lowercase_					:       Optional[int]     				=					use_token_type_ids
        lowercase_					:       Tuple     				=					use_labels
        lowercase_					:       Any     				=					vocab_size
        lowercase_					:       int     				=					hidden_size
        lowercase_					:       str     				=					num_hidden_layers
        lowercase_					:       Any     				=					num_attention_heads
        lowercase_					:       str     				=					intermediate_size
        lowercase_					:       Optional[Any]     				=					hidden_act
        lowercase_					:       str     				=					hidden_dropout_prob
        lowercase_					:       List[Any]     				=					attention_probs_dropout_prob
        lowercase_					:       Union[str, Any]     				=					max_position_embeddings
        lowercase_					:       Tuple     				=					type_vocab_size
        lowercase_					:       Optional[int]     				=					type_sequence_label_size
        lowercase_					:       List[Any]     				=					initializer_range
        lowercase_					:       Dict     				=					num_choices
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        lowercase_					:       Any     				=					ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size				)

        lowercase_					:       List[Any]     				=					None
        if self.use_attention_mask:
             lowercase_					:       str     				=					random_attention_mask([self.batch_size, self.seq_length]				)

        lowercase_					:       Optional[int]     				=					None
        if self.use_token_type_ids:
             lowercase_					:       Optional[int]     				=					ids_tensor([self.batch_size, self.seq_length]       ,				self.type_vocab_size				)

        lowercase_					:       Dict     				=					RobertaConfig(
            vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				type_vocab_size=self.type_vocab_size       ,				is_decoder=_a       ,				initializer_range=self.initializer_range       ,				)

        return config, input_ids, token_type_ids, attention_mask
   def 		_lowerCamelCase (self				)    ->     Tuple:
        lowercase_					:       List[str]     				=					self.prepare_config_and_inputs()
        lowercase_					:       str     				=					config_and_inputs
        lowercase_					:       Union[str, Any]     				=					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
        return config, inputs_dict





   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        lowercase_					:       Union[str, Any]     				=					self.prepare_config_and_inputs()
        lowercase_					:       Tuple     				=					config_and_inputs

        lowercase_					:       Dict     				=					True
        lowercase_					:       Union[str, Any]     				=					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
        lowercase_					:       Optional[Any]     				=					ids_tensor([self.batch_size, self.seq_length]       ,				vocab_size=2				)

        return (
            config,
            input_ids,
            token_type_ids,
            encoder_hidden_states,
            encoder_attention_mask,
        )



@require_flax
class       UpperCAmelCase__  (       __A					,     unittest.TestCase			):






   """simple docstring"""
   A							:       Optional[int]     =    True

   A							:       int     =    (
       (
           FlaxRobertaModel,
           FlaxRobertaForCausalLM,
           FlaxRobertaForMaskedLM,
           FlaxRobertaForSequenceClassification,
           FlaxRobertaForTokenClassification,
           FlaxRobertaForMultipleChoice,
           FlaxRobertaForQuestionAnswering,
       )
       if is_flax_available()
       else ()
   )
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        lowercase_					:       str     				=					FlaxRobertaModelTester(self				)





   @slow
   def 		_lowerCamelCase (self				)    ->     Tuple:

        for model_class_name in self.all_model_classes:
             lowercase_					:       Optional[Any]     				=					model_class_name.from_pretrained('roberta-base'       ,				from_pt=_a				)
             lowercase_					:       Optional[int]     				=					model(np.ones((1, 1)				)				)
             self.assertIsNotNone(_a				)





'''simple docstring'''




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin


class       UpperCAmelCase__  (       _snake_case					,     unittest.TestCase			):






   """simple docstring"""
   A							:       str     =    TransfoXLTokenizer
   A							:       Optional[int]     =    False
   A							:       List[str]     =    False
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        super().setUp()

        lowercase_					:       Any     				=					[
            '<unk>',
            '[CLS]',
            '[SEP]',
            'want',
            'unwanted',
            'wa',
            'un',
            'running',
            ',',
            'low',
            'l',
        ]
        lowercase_					:       Any     				=					os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['vocab_file']				)
        with open(self.vocab_file       ,				'w'       ,				encoding='utf-8'				) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)
   def 		_lowerCamelCase (self       ,				**_a				)    ->     Union[str, Any]:
        lowercase_					:       str     				=					True
        return TransfoXLTokenizer.from_pretrained(self.tmpdirname       ,				**_a				)
   def 		_lowerCamelCase (self       ,				_a				)    ->     Union[str, Any]:
        lowercase_					:       int     				=					'<unk> UNwanted , running'
        lowercase_					:       int     				=					'<unk> unwanted, running'
        return input_text, output_text
   def 		_lowerCamelCase (self				)    ->     Tuple:
        lowercase_					:       Any     				=					TransfoXLTokenizer(vocab_file=self.vocab_file       ,				lower_case=_a				)

        lowercase_					:       Optional[Any]     				=					tokenizer.tokenize('<unk> UNwanted , running'				)
        self.assertListEqual(_a       ,				['<unk>', 'unwanted', ',', 'running']				)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)       ,				[0, 4, 8, 7]				)
   def 		_lowerCamelCase (self				)    ->     Union[str, Any]:
        lowercase_					:       List[str]     				=					TransfoXLTokenizer(lower_case=_a				)

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '				)       ,				['hello', '!', 'how', 'are', 'you', '?']				)
   def 		_lowerCamelCase (self				)    ->     int:
        lowercase_					:       Dict     				=					TransfoXLTokenizer(lower_case=_a				)

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '				)       ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)
   def 		_lowerCamelCase (self				)    ->     Optional[Any]:
        lowercase_					:       Dict     				=					TransfoXLTokenizer(lower_case=_a				)
        lowercase_					:       int     				=					'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
        lowercase_					:       List[str]     				=					[
            'Hello',
            '(',
            'bracket',
            ')',
            'and',
            'side',
            '@-@',
            'scrolled',
            '[',
            'and',
            ']',
            'Henry',
            '\'s',
            '$',
            '5',
            '@,@',
            '000',
            'with',
            '3',
            '@.@',
            '34',
            'm',
            '.',
            'What',
            '\'s',
            'up',
            '!',
            '?',
        ]

        self.assertListEqual(tokenizer.tokenize(_a				)       ,				_a				)

        self.assertEqual(tokenizer.convert_tokens_to_string(_a				)       ,				_a				)





   def 		_lowerCamelCase (self				)    ->     Any:
        lowercase_					:       Optional[int]     				=					self.get_tokenizer()
        lowercase_					:       str     				=					len(_a				)

        tokenizer.add_tokens(['new1', 'new2']				)
        tokenizer.move_added_token('new1'       ,				1				)

        # Check that moved token is not copied (duplicate)
        self.assertEqual(len(_a				)       ,				original_len + 2				)
        # Check that token is moved to specified id
        self.assertEqual(tokenizer.encode('new1'				)       ,				[1]				)
        self.assertEqual(tokenizer.decode([1]				)       ,				'new1'				)




