





import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_lowerCamelCase   				=			data_utils.TransfoXLTokenizer
_lowerCamelCase   				=			data_utils.TransfoXLCorpus
_lowerCamelCase   				=			data_utils
_lowerCamelCase   				=			data_utils






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Dict					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							Tuple    ):
		if transfo_xl_dataset_file:
				# Convert a pre-processed corpus (see original TensorFlow repo)
				with open(UpperCamelCase__					,   """rb"""    ) as fp:
						SCREAMING_SNAKE_CASE__			=      pickle.load(UpperCamelCase__					,   encoding="""latin1"""    )
				# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
				SCREAMING_SNAKE_CASE__			=      pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
				print(f'''Save vocabulary to {pytorch_vocab_dump_path}'''    )
				SCREAMING_SNAKE_CASE__			=      corpus.vocab.__dict__
				torch.save(UpperCamelCase__					,   UpperCamelCase__    )

				SCREAMING_SNAKE_CASE__			=      corpus.__dict__
				corpus_dict_no_vocab.pop("""vocab"""					,   UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      pytorch_dump_folder_path + """/""" + CORPUS_NAME
				print(f'''Save dataset to {pytorch_dataset_dump_path}'''    )
				torch.save(UpperCamelCase__					,   UpperCamelCase__    )

		if tf_checkpoint_path:
				# Convert a pre-trained TensorFlow model
				SCREAMING_SNAKE_CASE__			=      os.path.abspath(UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      os.path.abspath(UpperCamelCase__    )

				print(f'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'''    )
				# Initialise PyTorch model
				if transfo_xl_config_file == "":
						SCREAMING_SNAKE_CASE__			=      TransfoXLConfig()
				else:
						SCREAMING_SNAKE_CASE__			=      TransfoXLConfig.from_json_file(UpperCamelCase__    )
				print(f'''Building PyTorch model from configuration: {config}'''    )
				SCREAMING_SNAKE_CASE__			=      TransfoXLLMHeadModel(UpperCamelCase__    )

				SCREAMING_SNAKE_CASE__			=      load_tf_weights_in_transfo_xl(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
				# Save pytorch-model
				SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   UpperCamelCase__    )
				print(f'''Save PyTorch model to {os.path.abspath(UpperCamelCase__    )}'''    )
				torch.save(model.state_dict()					,   UpperCamelCase__    )
				print(f'''Save configuration file to {os.path.abspath(UpperCamelCase__    )}'''    )
				with open(UpperCamelCase__					,   """w"""					,   encoding="""utf-8"""    ) as f:
						f.write(config.to_json_string()    )


if __name__ == "__main__":
	_lowerCamelCase   				=			argparse.ArgumentParser()
	parser.add_argument(
	    '--pytorch_dump_folder_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the folder to store the PyTorch model or dataset/vocab.',
	)
	parser.add_argument(
	    '--tf_checkpoint_path',
	    default='',
	    type=str,
	    help='An optional path to a TensorFlow checkpoint path to be converted.',
	)
	parser.add_argument(
	    '--transfo_xl_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained BERT model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--transfo_xl_dataset_file',
	    default='',
	    type=str,
	    help='An optional dataset file to be converted in a vocabulary.',
	)
	_lowerCamelCase   				=			parser.parse_args()
	convert_transfo_xl_checkpoint_to_pytorch(
	    args.tf_checkpoint_path,
	    args.transfo_xl_config_file,
	    args.pytorch_dump_folder_path,
	    args.transfo_xl_dataset_file,
	)






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import Accelerator
from accelerate.test_utils import execute_subprocess_async, require_multi_gpu
from accelerate.utils import patch_environment




class   UpperCamelCase_       (     unittest.TestCase    ):


							def      _snake_case  (							self    :Any		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      inspect.getfile(accelerate.test_utils		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_script.py"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(
									    mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_distributed_data_loop.py"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_ops.py"""]		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[Any]		)   ->   Tuple:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices.'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.test_file_path]
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Tuple		)   ->   Optional[Any]:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices.'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.operation_file_path]
									print(f'''Command: {cmd}'''		)
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', inspect.getfile(self.__class__		)]
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[int]		)   ->   str:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices, using 2 devices only'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.data_loop_file_path]
									with patch_environment(omp_num_threads=1							,				cuda_visible_devices="""0,1"""		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


if __name__ == "__main__":
	_lowerCamelCase   				=			Accelerator()
	_lowerCamelCase   				=			(accelerator.state.process_index + 2, 10)
	_lowerCamelCase   				=			torch.randint(0, 10, shape).to(accelerator.device)

	_lowerCamelCase   				=			''

	_lowerCamelCase   				=			accelerator.pad_across_processes(tensor)
	if tensora.shape[0] != accelerator.state.num_processes + 1:
		error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
	if not torch.equal(tensora[: accelerator.state.process_index + 2], tensor):
		error_msg += "Tensors have different values."
	if not torch.all(tensora[accelerator.state.process_index + 2 :] == 0):
		error_msg += "Padding was not done with the right value (0)."

	_lowerCamelCase   				=			accelerator.pad_across_processes(tensor, pad_first=True)
	if tensora.shape[0] != accelerator.state.num_processes + 1:
		error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
	_lowerCamelCase   				=			accelerator.state.num_processes - accelerator.state.process_index - 1
	if not torch.equal(tensora[index:], tensor):
		error_msg += "Tensors have different values."
	if not torch.all(tensora[:index] == 0):
		error_msg += "Padding was not done with the right value (0)."

	# Raise error at the end to make sure we don't stop at the first failure.
	if len(error_msg) > 0:
		raise ValueError(error_msg)