





import json
import os
import unittest

from transformers.models.biogpt.tokenization_biogpt import VOCAB_FILES_NAMES, BioGptTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin







class   _UpperCamelCase  (       _lowerCAmelCase						,       unittest.TestCase  ):







					'''simple docstring'''

					lowerCamelCase     :     int			    =       BioGptTokenizer
					lowerCamelCase     :     List[Any]			    =       False





					def        SCREAMING_SNAKE_CASE							(						self    :  List[str]		):






						'''simple docstring'''







						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						UpperCAmelCase_								=     [
						    """l""",
						    """o""",
						    """w""",
						    """e""",
						    """r""",
						    """s""",
						    """t""",
						    """i""",
						    """d""",
						    """n""",
						    """w</w>""",
						    """r</w>""",
						    """t</w>""",
						    """lo""",
						    """low""",
						    """er</w>""",
						    """low</w>""",
						    """lowest</w>""",
						    """newer</w>""",
						    """wider</w>""",
						    """<unk>""",
						]
						UpperCAmelCase_								=     dict(zip(_lowerCAmelCase ,					range(len(_lowerCAmelCase		)		)		)		)
						UpperCAmelCase_								=     ["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

						UpperCAmelCase_								=     os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["""vocab_file"""]		)
						UpperCAmelCase_								=     os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["""merges_file"""]		)
						with open(self.vocab_file ,					"""w"""		) as fp:
							fp.write(json.dumps(_lowerCAmelCase		)		)
						with open(self.merges_file ,					"""w"""		) as fp:
							fp.write("""\n""".join(_lowerCAmelCase		)		)





					def        SCREAMING_SNAKE_CASE							(						self    :  List[str] ,					__lowercase    :  Any		):






						'''simple docstring'''







						UpperCAmelCase_								=     """lower newer"""
						UpperCAmelCase_								=     """lower newer"""
						return input_text, output_text





					def        SCREAMING_SNAKE_CASE							(						self    :  Tuple		):






						'''simple docstring'''







						UpperCAmelCase_								=     BioGptTokenizer(self.vocab_file ,					self.merges_file		)

						UpperCAmelCase_								=     """lower"""
						UpperCAmelCase_								=     ["""low""", """er</w>"""]
						UpperCAmelCase_								=     tokenizer.tokenize(_lowerCAmelCase		)
						self.assertListEqual(_lowerCAmelCase ,					_lowerCAmelCase		)

						UpperCAmelCase_								=     tokens + ["""<unk>"""]
						UpperCAmelCase_								=     [14, 15, 20]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		) ,					_lowerCAmelCase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Any		):






						'''simple docstring'''







						UpperCAmelCase_								=     BioGptTokenizer.from_pretrained("""microsoft/biogpt"""		)

						UpperCAmelCase_								=     tokenizer.encode("""sequence builders""" ,					add_special_tokens=_lowerCAmelCase		)
						UpperCAmelCase_								=     tokenizer.encode("""multi-sequence build""" ,					add_special_tokens=_lowerCAmelCase		)

						UpperCAmelCase_								=     tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase		)
						UpperCAmelCase_								=     tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase ,					_lowerCAmelCase		)

						self.assertTrue(encoded_sentence == [2] + text		)
						self.assertTrue(encoded_pair == [2] + text + [2] + text_a		)









from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
				from transformers import (
				    AutoConfig,
				    BertConfig,
				    GPTaConfig,
				    TaConfig,
				    TFAutoModel,
				    TFAutoModelForCausalLM,
				    TFAutoModelForMaskedLM,
				    TFAutoModelForPreTraining,
				    TFAutoModelForQuestionAnswering,
				    TFAutoModelForSeqaSeqLM,
				    TFAutoModelForSequenceClassification,
				    TFAutoModelWithLMHead,
				    TFBertForMaskedLM,
				    TFBertForPreTraining,
				    TFBertForQuestionAnswering,
				    TFBertForSequenceClassification,
				    TFBertModel,
				    TFGPTaLMHeadModel,
				    TFRobertaForMaskedLM,
				    TFTaForConditionalGeneration,
				)
				from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
				from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
				from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
				from transformers import (
				    AutoModel,
				    AutoModelForCausalLM,
				    AutoModelForMaskedLM,
				    AutoModelForPreTraining,
				    AutoModelForQuestionAnswering,
				    AutoModelForSeqaSeqLM,
				    AutoModelForSequenceClassification,
				    AutoModelWithLMHead,
				    BertForMaskedLM,
				    BertForPreTraining,
				    BertForQuestionAnswering,
				    BertForSequenceClassification,
				    BertModel,
				    GPTaLMHeadModel,
				    RobertaForMaskedLM,
				    TaForConditionalGeneration,
				)







@is_pt_tf_cross_test
class   _UpperCamelCase  (       unittest.TestCase  ):







					'''simple docstring'''

					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModel.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModel.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForPreTraining.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForPreTraining.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Any		):






						'''simple docstring'''







						for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForCausalLM.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     TFAutoModelForCausalLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForCausalLM.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     AutoModelForCausalLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[Any]		):






						'''simple docstring'''







						for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelWithLMHead.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelWithLMHead.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  str		):






						'''simple docstring'''







						for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForMaskedLM.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     TFAutoModelForMaskedLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForMaskedLM.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     AutoModelForMaskedLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Union[str, Any]		):






						'''simple docstring'''







						for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForSeqaSeqLM.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     TFAutoModelForSeqaSeqLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForSeqaSeqLM.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     AutoModelForSeqaSeqLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[int]		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForSequenceClassification.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForSequenceClassification.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  str		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForQuestionAnswering.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForQuestionAnswering.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Tuple		):






						'''simple docstring'''







						UpperCAmelCase_								=     TFAutoModelWithLMHead.from_pretrained(__lowercase ,					from_pt=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)

						UpperCAmelCase_								=     AutoModelWithLMHead.from_pretrained(__lowercase ,					from_tf=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     TFAutoModelWithLMHead.from_pretrained(__lowercase ,					from_pt=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)

						UpperCAmelCase_								=     AutoModelWithLMHead.from_pretrained(__lowercase ,					from_tf=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)


