





import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm
				import torch_xla.debug.metrics as met







class   _UpperCamelCase  (       A_  ):







					'''simple docstring'''

					def __init__(						self    :  List[str] ,					*__lowercase    :  List[str] ,					__lowercase    :  str=None ,					__lowercase    :  Tuple=None ,					**__lowercase    :  Optional[int]		):






						'''simple docstring'''







						super().__init__(*__lowercase ,					**__lowercase		)
						UpperCAmelCase_								=     eval_examples
						UpperCAmelCase_								=     post_process_function





					def        SCREAMING_SNAKE_CASE							(						self    :  Any ,					__lowercase    :  Tuple=None ,					__lowercase    :  List[str]=None ,					__lowercase    :  List[str]=None ,					__lowercase    :  str = "eval"		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.eval_dataset if eval_dataset is None else eval_dataset
						UpperCAmelCase_								=     self.get_eval_dataloader(__lowercase		)
						UpperCAmelCase_								=     self.eval_examples if eval_examples is None else eval_examples

						# Temporarily disable metric computation, we will do it in the loop here.
						UpperCAmelCase_								=     self.compute_metrics
						UpperCAmelCase_								=     None
						UpperCAmelCase_								=     self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
						UpperCAmelCase_								=     time.time()
						try:
							UpperCAmelCase_								=     eval_loop(
							    __lowercase ,					description="""Evaluation""" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__lowercase ,					metric_key_prefix=__lowercase ,					)
						finally:
							UpperCAmelCase_								=     compute_metrics
						UpperCAmelCase_								=     self.args.eval_batch_size * self.args.world_size
						if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
							start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
						output.metrics.update(
						    speed_metrics(
						        __lowercase ,					__lowercase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size		) ,					)		)
						if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
							# Only the main node write the results by default
							UpperCAmelCase_								=     self.post_process_function(__lowercase ,					__lowercase ,					output.predictions		)
							UpperCAmelCase_								=     self.compute_metrics(__lowercase		)

							# Prefix all keys with metric_key_prefix + '_'
							for key in list(metrics.keys()		):
								if not key.startswith(F"""{metric_key_prefix}_"""		):
									UpperCAmelCase_								=     metrics.pop(__lowercase		)
							metrics.update(output.metrics		)
						else:
							UpperCAmelCase_								=     output.metrics

						if self.args.should_log:
							# Only the main node log the results by default
							self.log(__lowercase		)

						if self.args.tpu_metrics_debug or self.args.debug:
							# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
							xm.master_print(met.metrics_report()		)

						UpperCAmelCase_								=     self.callback_handler.on_evaluate(self.args ,					self.state ,					self.control ,					__lowercase		)
						return metrics





					def        SCREAMING_SNAKE_CASE							(						self    :  str ,					__lowercase    :  Optional[Any] ,					__lowercase    :  Optional[int] ,					__lowercase    :  Any=None ,					__lowercase    :  str = "test"		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.get_test_dataloader(__lowercase		)

						# Temporarily disable metric computation, we will do it in the loop here.
						UpperCAmelCase_								=     self.compute_metrics
						UpperCAmelCase_								=     None
						UpperCAmelCase_								=     self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
						UpperCAmelCase_								=     time.time()
						try:
							UpperCAmelCase_								=     eval_loop(
							    __lowercase ,					description="""Prediction""" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__lowercase ,					metric_key_prefix=__lowercase ,					)
						finally:
							UpperCAmelCase_								=     compute_metrics
						UpperCAmelCase_								=     self.args.eval_batch_size * self.args.world_size
						if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
							start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
						output.metrics.update(
						    speed_metrics(
						        __lowercase ,					__lowercase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size		) ,					)		)

						if self.post_process_function is None or self.compute_metrics is None:
							return output

						UpperCAmelCase_								=     self.post_process_function(__lowercase ,					__lowercase ,					output.predictions ,					"""predict"""		)
						UpperCAmelCase_								=     self.compute_metrics(__lowercase		)

						# Prefix all keys with metric_key_prefix + '_'
						for key in list(metrics.keys()		):
							if not key.startswith(F"""{metric_key_prefix}_"""		):
								UpperCAmelCase_								=     metrics.pop(__lowercase		)
						metrics.update(output.metrics		)
						return PredictionOutput(predictions=predictions.predictions ,					label_ids=predictions.label_ids ,					metrics=__lowercase		)









from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
				from transformers import (
				    AutoConfig,
				    BertConfig,
				    GPTaConfig,
				    TaConfig,
				    TFAutoModel,
				    TFAutoModelForCausalLM,
				    TFAutoModelForMaskedLM,
				    TFAutoModelForPreTraining,
				    TFAutoModelForQuestionAnswering,
				    TFAutoModelForSeqaSeqLM,
				    TFAutoModelForSequenceClassification,
				    TFAutoModelWithLMHead,
				    TFBertForMaskedLM,
				    TFBertForPreTraining,
				    TFBertForQuestionAnswering,
				    TFBertForSequenceClassification,
				    TFBertModel,
				    TFGPTaLMHeadModel,
				    TFRobertaForMaskedLM,
				    TFTaForConditionalGeneration,
				)
				from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
				from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
				from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
				from transformers import (
				    AutoModel,
				    AutoModelForCausalLM,
				    AutoModelForMaskedLM,
				    AutoModelForPreTraining,
				    AutoModelForQuestionAnswering,
				    AutoModelForSeqaSeqLM,
				    AutoModelForSequenceClassification,
				    AutoModelWithLMHead,
				    BertForMaskedLM,
				    BertForPreTraining,
				    BertForQuestionAnswering,
				    BertForSequenceClassification,
				    BertModel,
				    GPTaLMHeadModel,
				    RobertaForMaskedLM,
				    TaForConditionalGeneration,
				)







@is_pt_tf_cross_test
class   _UpperCamelCase  (       unittest.TestCase  ):







					'''simple docstring'''

					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModel.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModel.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForPreTraining.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForPreTraining.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Any		):






						'''simple docstring'''







						for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForCausalLM.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     TFAutoModelForCausalLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForCausalLM.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     AutoModelForCausalLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[Any]		):






						'''simple docstring'''







						for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelWithLMHead.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelWithLMHead.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  str		):






						'''simple docstring'''







						for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForMaskedLM.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     TFAutoModelForMaskedLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForMaskedLM.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     AutoModelForMaskedLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Union[str, Any]		):






						'''simple docstring'''







						for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForSeqaSeqLM.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     TFAutoModelForSeqaSeqLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForSeqaSeqLM.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							UpperCAmelCase_						,	UpperCAmelCase_								=     AutoModelForSeqaSeqLM.from_pretrained(
							    __lowercase ,					output_loading_info=__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[int]		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForSequenceClassification.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForSequenceClassification.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					@slow
					def        SCREAMING_SNAKE_CASE							(						self    :  str		):






						'''simple docstring'''







						for model_name in ["bert-base-uncased"]:
							UpperCAmelCase_								=     AutoConfig.from_pretrained(__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     TFAutoModelForQuestionAnswering.from_pretrained(__lowercase ,					from_pt=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)

							UpperCAmelCase_								=     AutoModelForQuestionAnswering.from_pretrained(__lowercase ,					from_tf=__lowercase		)
							self.assertIsNotNone(__lowercase		)
							self.assertIsInstance(__lowercase ,					__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Tuple		):






						'''simple docstring'''







						UpperCAmelCase_								=     TFAutoModelWithLMHead.from_pretrained(__lowercase ,					from_pt=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)

						UpperCAmelCase_								=     AutoModelWithLMHead.from_pretrained(__lowercase ,					from_tf=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     TFAutoModelWithLMHead.from_pretrained(__lowercase ,					from_pt=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)

						UpperCAmelCase_								=     AutoModelWithLMHead.from_pretrained(__lowercase ,					from_tf=__lowercase		)
						self.assertIsInstance(__lowercase ,					__lowercase		)
						self.assertEqual(model.num_parameters() ,					1_44_10		)
						self.assertEqual(model.num_parameters(only_trainable=__lowercase		) ,					1_44_10		)


