



'''simple docstring'''



import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
 from sagemaker import Session, TrainingJobAnalytics
 from sagemaker.huggingface import HuggingFace






@pytest.mark.skipif(
    literal_eval(os.getenv("""TEST_SAGEMAKER"""							,       """False""" ) ) is not True							,       reason="""Skipping test because should only be run when releasing minor transformers version"""							,       )
@pytest.mark.usefixtures("""sm_env""" )
@parameterized_class(
    [
        {
            """framework""": """pytorch""",
            """script""": """run_glue.py""",
            """model_name_or_path""": """distilbert-base-cased""",
            """instance_type""": """ml.g4dn.xlarge""",
            """results""": {"""train_runtime""": 6_5_0, """eval_accuracy""": 0.6, """eval_loss""": 0.9},
        },
        {
            """framework""": """tensorflow""",
            """script""": """run_tf.py""",
            """model_name_or_path""": """distilbert-base-cased""",
            """instance_type""": """ml.g4dn.xlarge""",
            """results""": {"""train_runtime""": 6_0_0, """eval_accuracy""": 0.3, """eval_loss""": 0.9},
        },
    ] )
class 					UpperCAmelCase__	(		unittest.TestCase ):



     def        UpperCAmelCase_ (    self     )     ->			Any:




      if self.framework == "pytorch":
       subprocess.run(
           F'''cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'''.split()			,				encoding="utf-8"			,				check=A_			,				)
      assert hasattr(self			,				"env"     )



     def        UpperCAmelCase_ (    self			,				UpperCamelCase=1     )     ->			Optional[Any]:
      return HuggingFace(
          entry_point=self.script			,				source_dir=self.env.test_path			,				role=self.env.role			,				image_uri=self.env.image_uri			,				base_job_name=F'''{self.env.base_job_name}-single'''			,				instance_count=A_			,				instance_type=self.instance_type			,				debugger_hook_config=A_			,				hyperparameters={**self.env.hyperparameters, "model_name_or_path": self.model_name_or_path}			,				metric_definitions=self.env.metric_definitions			,				py_version="py36"			,				)



     def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			Union[str, Any]:
      TrainingJobAnalytics(A_     ).export_csv(F'''{self.env.test_path}/{job_name}_metrics.csv'''     )



     def        UpperCAmelCase_ (    self     )     ->			int:
      __lowerCAmelCase												=  self.create_estimator()

      # run training
      estimator.fit()

      # result dataframe
      __lowerCAmelCase												=  TrainingJobAnalytics(estimator.latest_training_job.name     ).dataframe()

      # extract kpis
      __lowerCAmelCase												=  list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]     )
      __lowerCAmelCase												=  list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]     )
      # get train time from SageMaker job, this includes starting, preprocessing, stopping
      __lowerCAmelCase												=  (
          Session().describe_training_job(estimator.latest_training_job.name     ).get("TrainingTimeInSeconds"			,				99_9999     )
      )

      # assert kpis
      assert train_runtime <= self.results["train_runtime"]
      assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy     )
      assert all(t <= self.results["eval_loss"] for t in eval_loss     )

      # dump tests result into json file to share in PR
      with open(F'''{estimator.latest_training_job.name}.json'''			,				"w"     ) as outfile:
       json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}			,				A_     )




'''simple docstring'''



import os
import sys
import unittest


lowerCAmelCase	:		str				=   os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import get_test_info  # noqa: E402
from get_test_info import (  # noqa: E402
    get_model_to_test_mapping,
    get_model_to_tester_mapping,
    get_test_to_tester_mapping,
)


lowerCAmelCase	:		Tuple				=   os.path.join('''tests''', '''models''', '''bert''', '''test_modeling_bert.py''')
lowerCAmelCase	:		Tuple				=   os.path.join('''tests''', '''models''', '''blip''', '''test_modeling_blip.py''')






class 					UpperCAmelCase__	(		unittest.TestCase ):



					def        UpperCAmelCase_ (    self     )     ->			str:
						__lowerCAmelCase												=  get_test_to_tester_mapping(UpperCamelCase     )
						__lowerCAmelCase												=  get_test_to_tester_mapping(UpperCamelCase     )

						__lowerCAmelCase												=  {"BertModelTest": "BertModelTester"}

						__lowerCAmelCase												=  {
						    "BlipModelTest": "BlipModelTester",
						    "BlipTextImageModelTest": "BlipTextImageModelsModelTester",
						    "BlipTextModelTest": "BlipTextModelTester",
						    "BlipTextRetrievalModelTest": "BlipTextRetrievalModelTester",
						    "BlipVQAModelTest": "BlipVQAModelTester",
						    "BlipVisionModelTest": "BlipVisionModelTester",
						}

						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )
						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )



					def        UpperCAmelCase_ (    self     )     ->			List[Any]:
						__lowerCAmelCase												=  get_model_to_test_mapping(UpperCamelCase     )
						__lowerCAmelCase												=  get_model_to_test_mapping(UpperCamelCase     )

						__lowerCAmelCase												=  {
						    "BertForMaskedLM": ["BertModelTest"],
						    "BertForMultipleChoice": ["BertModelTest"],
						    "BertForNextSentencePrediction": ["BertModelTest"],
						    "BertForPreTraining": ["BertModelTest"],
						    "BertForQuestionAnswering": ["BertModelTest"],
						    "BertForSequenceClassification": ["BertModelTest"],
						    "BertForTokenClassification": ["BertModelTest"],
						    "BertLMHeadModel": ["BertModelTest"],
						    "BertModel": ["BertModelTest"],
						}

						__lowerCAmelCase												=  {
						    "BlipForConditionalGeneration": ["BlipTextImageModelTest"],
						    "BlipForImageTextRetrieval": ["BlipTextRetrievalModelTest"],
						    "BlipForQuestionAnswering": ["BlipVQAModelTest"],
						    "BlipModel": ["BlipModelTest"],
						    "BlipTextModel": ["BlipTextModelTest"],
						    "BlipVisionModel": ["BlipVisionModelTest"],
						}

						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )
						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )



					def        UpperCAmelCase_ (    self     )     ->			str:
						__lowerCAmelCase												=  get_model_to_tester_mapping(UpperCamelCase     )
						__lowerCAmelCase												=  get_model_to_tester_mapping(UpperCamelCase     )

						__lowerCAmelCase												=  {
						    "BertForMaskedLM": ["BertModelTester"],
						    "BertForMultipleChoice": ["BertModelTester"],
						    "BertForNextSentencePrediction": ["BertModelTester"],
						    "BertForPreTraining": ["BertModelTester"],
						    "BertForQuestionAnswering": ["BertModelTester"],
						    "BertForSequenceClassification": ["BertModelTester"],
						    "BertForTokenClassification": ["BertModelTester"],
						    "BertLMHeadModel": ["BertModelTester"],
						    "BertModel": ["BertModelTester"],
						}

						__lowerCAmelCase												=  {
						    "BlipForConditionalGeneration": ["BlipTextImageModelsModelTester"],
						    "BlipForImageTextRetrieval": ["BlipTextRetrievalModelTester"],
						    "BlipForQuestionAnswering": ["BlipVQAModelTester"],
						    "BlipModel": ["BlipModelTester"],
						    "BlipTextModel": ["BlipTextModelTester"],
						    "BlipVisionModel": ["BlipVisionModelTester"],
						}

						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )
						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )