


import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch






class 						__a (     unittest.TestCase				):
	SCREAMING_SNAKE_CASE       =       JukeboxTokenizer
	SCREAMING_SNAKE_CASE       =       {
	    "artist": "Zac Brown Band",
	    "genres": "Country",
	    "lyrics": "I met a traveller from an antique land,\n        Who said \"Two vast and trunkless legs of stone\n        Stand in the desert. . . . Near them, on the sand,\n        Half sunk a shattered visage lies, whose frown,\n        And wrinkled lip, and sneer of cold command,\n        Tell that its sculptor well those passions read\n        Which yet survive, stamped on these lifeless things,\n        The hand that mocked them, and the heart that fed;\n        And on the pedestal, these words appear:\n        My name is Ozymandias, King of Kings;\n        Look on my Works, ye Mighty, and despair!\n        Nothing beside remains. Round the decay\n        Of that colossal Wreck, boundless and bare\n        The lone and level sands stretch far away\n        ",
	}



	@require_torch
	def 		UpperCamelCase							(						self							:    int)->							str:
						import torch

						__lowerCAmelCase	   =JukeboxTokenizer.from_pretrained("""openai/jukebox-1b-lyrics""")
						__lowerCAmelCase	   =tokenizer(**self.metas)["input_ids"]
						# fmt: off
						__lowerCAmelCase	   =[
						    torch.tensor([[
						        0, 0, 0, 71_69, 5_07, 9, 76, 39, 31, 46, 76, 27,
						        76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
						        44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
						        47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
						        76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
						        30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
						        27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
						        45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
						        41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
						        76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
						        76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
						        64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
						        30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
						        27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
						        34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
						        27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
						        41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
						        76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
						        44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
						        76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
						        32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
						        40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
						        45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
						        31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
						        45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
						        76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
						        34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
						        31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
						        40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
						        38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
						        76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
						        41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
						        27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
						        46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
						        76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
						        41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
						        46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
						        41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
						        40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
						        27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
						        76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
						        76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
						        41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
						        76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
						        27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
						        34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
						        44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
						        40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
						        46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
						        38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
						        40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
						        27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
						        76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
						        76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
						        76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
						        76, 76]]),
						    torch.tensor([[0, 0, 0, 10_69, 11]]),
						    torch.tensor([[0, 0, 0, 10_69, 11]]),
						]
						# fmt: on
						self.assertTrue(torch.allclose(tokens[0]				,	EXPECTED_OUTPUT[0]))
						self.assertTrue(torch.allclose(tokens[1]				,	EXPECTED_OUTPUT[1]))
						self.assertTrue(torch.allclose(tokens[2]				,	EXPECTED_OUTPUT[2]))



	@require_torch
	def 		UpperCamelCase							(						self							:    Union[str, Any])->							Dict:
						import torch

						__lowerCAmelCase	   =JukeboxTokenizer.from_pretrained("""openai/jukebox-5b-lyrics""")
						__lowerCAmelCase	   =tokenizer(**self.metas)["input_ids"]
						# fmt: off
						__lowerCAmelCase	   =[
						    torch.tensor([[
						        0, 0, 0, 10_69, 11, -1, -1, -1, -1, 9, 77, 39,
						        31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
						        31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
						        40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
						        79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
						        77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
						        27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
						        37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
						        32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
						        77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
						        77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
						        46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
						        77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
						        77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
						        77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
						        77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
						        64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
						        40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
						        40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
						        38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
						        31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
						        41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
						        46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
						        41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
						        31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
						        31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
						        23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
						        44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
						        31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
						        38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
						        40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
						        27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
						        31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
						        34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
						        31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
						        1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
						        31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
						        45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
						        31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
						        15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
						        11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
						        45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
						        41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
						        44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
						        46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
						        27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
						        35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
						        77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
						        31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
						        41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
						        77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
						        40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
						        77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
						        27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
						        77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
						        77, 77, 77, 77, 77, 77]]),
						    torch.tensor([[0, 0, 0, 10_69, 11, -1, -1, -1, -1]]),
						    torch.tensor([[0, 0, 0, 10_69, 11, -1, -1, -1, -1]]),
						]
						# fmt: on
						self.assertTrue(torch.allclose(tokens[0]				,	EXPECTED_OUTPUT[0]))
						self.assertTrue(torch.allclose(tokens[1]				,	EXPECTED_OUTPUT[1]))
						self.assertTrue(torch.allclose(tokens[2]				,	EXPECTED_OUTPUT[2]))









import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
				from sagemaker import Session, TrainingJobAnalytics
				from sagemaker.huggingface import HuggingFace






@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"	,    "False"				)				) is not True	,    reason="Skipping test because should only be run when releasing minor transformers version"	,    )
@pytest.mark.usefixtures("sm_env"				)
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 650, "eval_accuracy": 0.7, "eval_loss": 0.6},
        },
        {
            "framework": "pytorch",
            "script": "run_ddp.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.7, "eval_loss": 0.6},
        },
        {
            "framework": "tensorflow",
            "script": "run_tf_dist.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.6, "eval_loss": 0.7},
        },
    ]				)
class 						__a (     unittest.TestCase				):



	def 		UpperCamelCase							(						self							:    Dict)->							Any:
						if self.framework == "pytorch":
											subprocess.run(
											    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()				,	encoding="""utf-8"""				,	check=snake_case_				,	)
						assert hasattr(self				,	"""env""")



	def 		UpperCamelCase							(						self							:    Dict				,	snake_case_							:    str)->							Dict:
						__lowerCAmelCase	   =F"""{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"""
						# distributed data settings
						__lowerCAmelCase	   ={"""smdistributed""": {"""dataparallel""": {"""enabled""": True}}} if self.script != """run_ddp.py""" else None

						# creates estimator
						return HuggingFace(
						    entry_point=self.script				,	source_dir=self.env.test_path				,	role=self.env.role				,	image_uri=self.env.image_uri				,	base_job_name=snake_case_				,	instance_count=snake_case_				,	instance_type=self.instance_type				,	debugger_hook_config=snake_case_				,	hyperparameters={**self.env.distributed_hyperparameters, """model_name_or_path""": self.model_name_or_path}				,	metric_definitions=self.env.metric_definitions				,	distribution=snake_case_				,	py_version="""py36"""				,	)



	def 		UpperCamelCase							(						self							:    int				,	snake_case_							:    List[Any])->							int:
						TrainingJobAnalytics(snake_case_).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv""")



	@parameterized.expand([(2,)])
	def 		UpperCamelCase							(						self							:    Any				,	snake_case_							:    Dict)->							Optional[Any]:
						# create estimator
						__lowerCAmelCase	   =self.create_estimator(snake_case_)

						# run training
						estimator.fit()

						# result dataframe
						__lowerCAmelCase	   =TrainingJobAnalytics(estimator.latest_training_job.name).dataframe()

						# extract kpis
						__lowerCAmelCase	   =list(result_metrics_df[result_metrics_df.metric_name == """eval_accuracy"""]["""value"""])
						__lowerCAmelCase	   =list(result_metrics_df[result_metrics_df.metric_name == """eval_loss"""]["""value"""])
						# get train time from SageMaker job, this includes starting, preprocessing, stopping
						__lowerCAmelCase	   =(
						    Session().describe_training_job(estimator.latest_training_job.name).get("""TrainingTimeInSeconds"""				,	99_99_99)
						)

						# assert kpis
						assert train_runtime <= self.results["train_runtime"]
						assert all(t >= self.results["""eval_accuracy"""] for t in eval_accuracy)
						assert all(t <= self.results["""eval_loss"""] for t in eval_loss)

						# dump tests result into json file to share in PR
						with open(F"""{estimator.latest_training_job.name}.json"""				,	"""w""") as outfile:
											json.dump({"""train_time""": train_runtime, """eval_accuracy""": eval_accuracy, """eval_loss""": eval_loss}				,	snake_case_)




