





'''simple docstring'''

import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin





class lowerCamelCase						(a__						,      unittest.TestCase					):
  _lowercase  :	List[str]       					=				XLMTokenizer
  _lowercase  :	List[str]       					=				False







  def 				UpperCAmelCase_				(		self		)		->						Union[str, Any]:






        """simple docstring"""





        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        _snake_case		:	Union[str, Any]					     =     [
            '''l''',
            '''o''',
            '''w''',
            '''e''',
            '''r''',
            '''s''',
            '''t''',
            '''i''',
            '''d''',
            '''n''',
            '''w</w>''',
            '''r</w>''',
            '''t</w>''',
            '''lo''',
            '''low''',
            '''er</w>''',
            '''low</w>''',
            '''lowest</w>''',
            '''newer</w>''',
            '''wider</w>''',
            '''<unk>''',
        ]
        _snake_case		:	Any					     =     dict(zip(lowercase__     ,     range(len(lowercase__		)		)		)		)
        _snake_case		:	List[Any]					     =     ['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

        _snake_case		:	Any					     =     os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES['''vocab_file''']		)
        _snake_case		:	str					     =     os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES['''merges_file''']		)
        with open(self.vocab_file     ,     '''w'''		) as fp:
              fp.write(json.dumps(lowercase__		)		)
        with open(self.merges_file     ,     '''w'''		) as fp:
              fp.write('''\n'''.join(lowercase__		)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Optional[Any]:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     '''lower newer'''
        _snake_case		:	Dict					     =     '''lower newer'''
        return input_text, output_text







  def 				UpperCAmelCase_				(		self		)		->						Tuple:






        """simple docstring"""





        _snake_case		:	Tuple					     =     XLMTokenizer(self.vocab_file     ,     self.merges_file		)

        _snake_case		:	Optional[int]					     =     '''lower'''
        _snake_case		:	str					     =     ['''low''', '''er</w>''']
        _snake_case		:	Optional[Any]					     =     tokenizer.tokenize(lowercase__		)
        self.assertListEqual(lowercase__     ,     lowercase__		)

        _snake_case		:	Tuple					     =     tokens + ['''<unk>''']
        _snake_case		:	Dict					     =     [14, 15, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__		)     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Union[str, Any]:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     XLMTokenizer.from_pretrained('''xlm-mlm-en-2048'''		)

        _snake_case		:	str					     =     tokenizer.encode('''sequence builders'''     ,     add_special_tokens=lowercase__		)
        _snake_case		:	Tuple					     =     tokenizer.encode('''multi-sequence build'''     ,     add_special_tokens=lowercase__		)

        _snake_case		:	Tuple					     =     tokenizer.build_inputs_with_special_tokens(lowercase__		)
        _snake_case		:	List[str]					     =     tokenizer.build_inputs_with_special_tokens(lowercase__     ,     lowercase__		)

        assert encoded_sentence == [0] + text + [1]
        assert encoded_pair == [0] + text + [1] + text_a + [1]









'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
   from transformers import (
       AutoConfig,
       BertConfig,
       GPTaConfig,
       TaConfig,
       TFAutoModel,
       TFAutoModelForCausalLM,
       TFAutoModelForMaskedLM,
       TFAutoModelForPreTraining,
       TFAutoModelForQuestionAnswering,
       TFAutoModelForSeqaSeqLM,
       TFAutoModelForSequenceClassification,
       TFAutoModelWithLMHead,
       TFBertForMaskedLM,
       TFBertForPreTraining,
       TFBertForQuestionAnswering,
       TFBertForSequenceClassification,
       TFBertModel,
       TFGPTaLMHeadModel,
       TFRobertaForMaskedLM,
       TFTaForConditionalGeneration,
   )
   from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
   from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
   from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
   from transformers import (
       AutoModel,
       AutoModelForCausalLM,
       AutoModelForMaskedLM,
       AutoModelForPreTraining,
       AutoModelForQuestionAnswering,
       AutoModelForSeqaSeqLM,
       AutoModelForSequenceClassification,
       AutoModelWithLMHead,
       BertForMaskedLM,
       BertForPreTraining,
       BertForQuestionAnswering,
       BertForSequenceClassification,
       BertModel,
       GPTaLMHeadModel,
       RobertaForMaskedLM,
       TaForConditionalGeneration,
   )





@is_pt_tf_cross_test
class lowerCamelCase						(unittest.TestCase					):







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Union[str, Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Any					     =     TFAutoModel.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	str					     =     AutoModel.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[str]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Optional[Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Dict					     =     TFAutoModelForPreTraining.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     AutoModelForPreTraining.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Union[str, Any]:






        """simple docstring"""





        for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	Optional[int]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	List[Any]					     =     TFAutoModelForCausalLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	Tuple					     =     TFAutoModelForCausalLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Optional[int]					     =     AutoModelForCausalLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Optional[Any]					     =     AutoModelForCausalLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Optional[Any]:






        """simple docstring"""





        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	int					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[str]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Union[str, Any]					     =     TFAutoModelForMaskedLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	List[str]					     =     TFAutoModelForMaskedLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	int					     =     AutoModelForMaskedLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Optional[int]					     =     AutoModelForMaskedLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[str]:






        """simple docstring"""





        for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[str]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     TFAutoModelForSeqaSeqLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	List[str]					     =     TFAutoModelForSeqaSeqLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	List[str]					     =     AutoModelForSeqaSeqLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Dict					     =     AutoModelForSeqaSeqLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Dict:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Any					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Any					     =     TFAutoModelForSequenceClassification.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Dict					     =     AutoModelForSequenceClassification.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Optional[int]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	str					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	str					     =     TFAutoModelForQuestionAnswering.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Union[str, Any]					     =     AutoModelForQuestionAnswering.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	Union[str, Any]					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)

        _snake_case		:	Tuple					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	List[str]					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)

        _snake_case		:	int					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)


