
"""simple docstring"""


import pytest

from datasets.splits import SplitDict, SplitInfo
from datasets.utils.py_utils import asdict






@pytest.mark.parametrize(
    'split_dict'	,      [
        SplitDict(),
        SplitDict({'train': SplitInfo(name='train'	,      num_bytes=1337	,      num_examples=42	,      dataset_name='my_dataset'     )}     ),
        SplitDict({'train': SplitInfo(name='train'	,      num_bytes=1337	,      num_examples=42     )}     ),
        SplitDict({'train': SplitInfo()}     ),
    ]	,      )
def       _snake_case				(		snake_case__   :     SplitDict     ):
	A						   =   split_dict._to_yaml_list()
	assert len(snake_case__     ) == len(snake_case__     )
	A						   =   SplitDict._from_yaml_list(snake_case__     )
	for split_name, split_info in split_dict.items():
		# dataset_name field is deprecated, and is therefore not part of the YAML dump
		A						   =   None
		# the split name of split_dict takes over the name of the split info object
		A						   =   split_name
	assert split_dict == reloaded






@pytest.mark.parametrize(
    'split_info'	,      [SplitInfo(), SplitInfo(dataset_name=snake_case__     ), SplitInfo(dataset_name='my_dataset'     )]     )
def       _snake_case				(		snake_case__   :     Dict     ):
	# For backward compatibility, we need asdict(split_dict) to return split info dictrionaries with the "dataset_name"
	# field even if it's deprecated. This way old versionso of `datasets` can still reload dataset_infos.json files
	A						   =   asdict(SplitDict({'train': split_info}     )     )
	assert "dataset_name" in split_dict_asdict["train"]
	assert split_dict_asdict["train"]["dataset_name"] == split_info.dataset_name

"""simple docstring"""


import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_lowercase							   =				data_utils.TransfoXLTokenizer
_lowercase							   =				data_utils.TransfoXLCorpus
_lowercase							   =				data_utils
_lowercase							   =				data_utils






def       _snake_case				(		snake_case__   :     Union[str, Any]	,      snake_case__   :     Dict	,      snake_case__   :     Tuple	,      snake_case__   :     int     ):
	if transfo_xl_dataset_file:
		# Convert a pre-processed corpus (see original TensorFlow repo)
		with open(snake_case__	,      'rb'     ) as fp:
			A						   =   pickle.load(snake_case__	,      encoding='latin1'     )
		# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
		A						   =   pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
		print(F'Save vocabulary to {pytorch_vocab_dump_path}'     )
		A						   =   corpus.vocab.__dict__
		torch.save(snake_case__	,      snake_case__     )

		A						   =   corpus.__dict__
		corpus_dict_no_vocab.pop('vocab'	,      snake_case__     )
		A						   =   pytorch_dump_folder_path + '/' + CORPUS_NAME
		print(F'Save dataset to {pytorch_dataset_dump_path}'     )
		torch.save(snake_case__	,      snake_case__     )

	if tf_checkpoint_path:
		# Convert a pre-trained TensorFlow model
		A						   =   os.path.abspath(snake_case__     )
		A						   =   os.path.abspath(snake_case__     )

		print(F'Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'     )
		# Initialise PyTorch model
		if transfo_xl_config_file == "":
			A						   =   TransfoXLConfig()
		else:
			A						   =   TransfoXLConfig.from_json_file(snake_case__     )
		print(F'Building PyTorch model from configuration: {config}'     )
		A						   =   TransfoXLLMHeadModel(snake_case__     )

		A						   =   load_tf_weights_in_transfo_xl(snake_case__	,      snake_case__	,      snake_case__     )
		# Save pytorch-model
		A						   =   os.path.join(snake_case__	,      snake_case__     )
		A						   =   os.path.join(snake_case__	,      snake_case__     )
		print(F'Save PyTorch model to {os.path.abspath(snake_case__     )}'     )
		torch.save(model.state_dict()	,      snake_case__     )
		print(F'Save configuration file to {os.path.abspath(snake_case__     )}'     )
		with open(snake_case__	,      'w'	,      encoding='utf-8'     ) as f:
			f.write(config.to_json_string()     )


if __name__ == "__main__":
		_lowercase							   =				argparse.ArgumentParser()
		parser.add_argument(
		    '''--pytorch_dump_folder_path''',
		    default=None,
		    type=str,
		    required=True,
		    help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
		)
		parser.add_argument(
		    '''--tf_checkpoint_path''',
		    default='''''',
		    type=str,
		    help='''An optional path to a TensorFlow checkpoint path to be converted.''',
		)
		parser.add_argument(
		    '''--transfo_xl_config_file''',
		    default='''''',
		    type=str,
		    help=(
		        '''An optional config json file corresponding to the pre-trained BERT model. \n'''
		        '''This specifies the model architecture.'''
		    ),
		)
		parser.add_argument(
		    '''--transfo_xl_dataset_file''',
		    default='''''',
		    type=str,
		    help='''An optional dataset file to be converted in a vocabulary.''',
		)
		_lowercase							   =				parser.parse_args()
		convert_transfo_xl_checkpoint_to_pytorch(
		    args.tf_checkpoint_path,
		    args.transfo_xl_config_file,
		    args.pytorch_dump_folder_path,
		    args.transfo_xl_dataset_file,
		)