





import os

import pytest

from datasets import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
)


_lowerCamelCase   				=			pytest.mark.integration


@pytest.mark.parametrize("""path"""					,   ["""paws""", """csv"""]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							Any    ):
		inspect_dataset(lowerCAmelCase__					,   lowerCAmelCase__    )
		SCREAMING_SNAKE_CASE__			=      path + """.py"""
		assert script_name in os.listdir(lowerCAmelCase__    )
		assert "__pycache__" not in os.listdir(lowerCAmelCase__    )


@pytest.mark.filterwarnings("""ignore:inspect_metric is deprecated:FutureWarning"""    )
@pytest.mark.filterwarnings("""ignore:metric_module_factory is deprecated:FutureWarning"""    )
@pytest.mark.parametrize("""path"""					,   ["""accuracy"""]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any					,   UpperCamelCase__:							Optional[int]    ):
		inspect_metric(lowerCAmelCase__					,   lowerCAmelCase__    )
		SCREAMING_SNAKE_CASE__			=      path + """.py"""
		assert script_name in os.listdir(lowerCAmelCase__    )
		assert "__pycache__" not in os.listdir(lowerCAmelCase__    )


@pytest.mark.parametrize(
    """path, config_name, expected_splits"""					,   [
        ("""squad""", """plain_text""", ["""train""", """validation"""]),
        ("""dalle-mini/wit""", """dalle-mini--wit""", ["""train"""]),
        ("""paws""", """labeled_final""", ["""train""", """test""", """validation"""]),
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any					,   UpperCamelCase__:							Dict					,   UpperCamelCase__:							List[str]    ):
		SCREAMING_SNAKE_CASE__			=      get_dataset_config_info(lowerCAmelCase__					,   config_name=lowerCAmelCase__    )
		assert info.config_name == config_name
		assert list(info.splits.keys()    ) == expected_splits


@pytest.mark.parametrize(
    """path, config_name, expected_exception"""					,   [
        ("""paws""", None, ValueError),
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							str					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Any    ):
		with pytest.raises(lowerCAmelCase__    ):
				get_dataset_config_info(lowerCAmelCase__					,   config_name=lowerCAmelCase__    )


@pytest.mark.parametrize(
    """path, expected"""					,   [
        ("""squad""", """plain_text"""),
        ("""acronym_identification""", """default"""),
        ("""lhoestq/squad""", """plain_text"""),
        ("""lhoestq/test""", """default"""),
        ("""lhoestq/demo1""", """lhoestq--demo1"""),
        ("""dalle-mini/wit""", """dalle-mini--wit"""),
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Dict					,   UpperCamelCase__:							Any    ):
		SCREAMING_SNAKE_CASE__			=      get_dataset_config_names(lowerCAmelCase__    )
		assert expected in config_names


@pytest.mark.parametrize(
    """path, expected_configs, expected_splits_in_first_config"""					,   [
        ("""squad""", ["""plain_text"""], ["""train""", """validation"""]),
        ("""dalle-mini/wit""", ["""dalle-mini--wit"""], ["""train"""]),
        ("""paws""", ["""labeled_final""", """labeled_swap""", """unlabeled_final"""], ["""train""", """test""", """validation"""]),
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							str					,   UpperCamelCase__:							List[Any]					,   UpperCamelCase__:							int    ):
		SCREAMING_SNAKE_CASE__			=      get_dataset_infos(lowerCAmelCase__    )
		assert list(infos.keys()    ) == expected_configs
		SCREAMING_SNAKE_CASE__			=      expected_configs[0]
		assert expected_config in infos
		SCREAMING_SNAKE_CASE__			=      infos[expected_config]
		assert info.config_name == expected_config
		assert list(info.splits.keys()    ) == expected_splits_in_first_config


@pytest.mark.parametrize(
    """path, expected_config, expected_splits"""					,   [
        ("""squad""", """plain_text""", ["""train""", """validation"""]),
        ("""dalle-mini/wit""", """dalle-mini--wit""", ["""train"""]),
        ("""paws""", """labeled_final""", ["""train""", """test""", """validation"""]),
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							Union[str, Any]					,   UpperCamelCase__:							Any    ):
		SCREAMING_SNAKE_CASE__			=      get_dataset_infos(lowerCAmelCase__    )
		assert expected_config in infos
		SCREAMING_SNAKE_CASE__			=      infos[expected_config]
		assert info.config_name == expected_config
		assert list(info.splits.keys()    ) == expected_splits


@pytest.mark.parametrize(
    """path, config_name, expected_exception"""					,   [
        ("""paws""", None, ValueError),
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[Any]					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							List[Any]    ):
		with pytest.raises(lowerCAmelCase__    ):
				get_dataset_split_names(lowerCAmelCase__					,   config_name=lowerCAmelCase__    )






import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_lowerCamelCase   				=			data_utils.TransfoXLTokenizer
_lowerCamelCase   				=			data_utils.TransfoXLCorpus
_lowerCamelCase   				=			data_utils
_lowerCamelCase   				=			data_utils






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Dict					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							Tuple    ):
		if transfo_xl_dataset_file:
				# Convert a pre-processed corpus (see original TensorFlow repo)
				with open(UpperCamelCase__					,   """rb"""    ) as fp:
						SCREAMING_SNAKE_CASE__			=      pickle.load(UpperCamelCase__					,   encoding="""latin1"""    )
				# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
				SCREAMING_SNAKE_CASE__			=      pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
				print(f'''Save vocabulary to {pytorch_vocab_dump_path}'''    )
				SCREAMING_SNAKE_CASE__			=      corpus.vocab.__dict__
				torch.save(UpperCamelCase__					,   UpperCamelCase__    )

				SCREAMING_SNAKE_CASE__			=      corpus.__dict__
				corpus_dict_no_vocab.pop("""vocab"""					,   UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      pytorch_dump_folder_path + """/""" + CORPUS_NAME
				print(f'''Save dataset to {pytorch_dataset_dump_path}'''    )
				torch.save(UpperCamelCase__					,   UpperCamelCase__    )

		if tf_checkpoint_path:
				# Convert a pre-trained TensorFlow model
				SCREAMING_SNAKE_CASE__			=      os.path.abspath(UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      os.path.abspath(UpperCamelCase__    )

				print(f'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'''    )
				# Initialise PyTorch model
				if transfo_xl_config_file == "":
						SCREAMING_SNAKE_CASE__			=      TransfoXLConfig()
				else:
						SCREAMING_SNAKE_CASE__			=      TransfoXLConfig.from_json_file(UpperCamelCase__    )
				print(f'''Building PyTorch model from configuration: {config}'''    )
				SCREAMING_SNAKE_CASE__			=      TransfoXLLMHeadModel(UpperCamelCase__    )

				SCREAMING_SNAKE_CASE__			=      load_tf_weights_in_transfo_xl(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
				# Save pytorch-model
				SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   UpperCamelCase__    )
				print(f'''Save PyTorch model to {os.path.abspath(UpperCamelCase__    )}'''    )
				torch.save(model.state_dict()					,   UpperCamelCase__    )
				print(f'''Save configuration file to {os.path.abspath(UpperCamelCase__    )}'''    )
				with open(UpperCamelCase__					,   """w"""					,   encoding="""utf-8"""    ) as f:
						f.write(config.to_json_string()    )


if __name__ == "__main__":
	_lowerCamelCase   				=			argparse.ArgumentParser()
	parser.add_argument(
	    '--pytorch_dump_folder_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the folder to store the PyTorch model or dataset/vocab.',
	)
	parser.add_argument(
	    '--tf_checkpoint_path',
	    default='',
	    type=str,
	    help='An optional path to a TensorFlow checkpoint path to be converted.',
	)
	parser.add_argument(
	    '--transfo_xl_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained BERT model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--transfo_xl_dataset_file',
	    default='',
	    type=str,
	    help='An optional dataset file to be converted in a vocabulary.',
	)
	_lowerCamelCase   				=			parser.parse_args()
	convert_transfo_xl_checkpoint_to_pytorch(
	    args.tf_checkpoint_path,
	    args.transfo_xl_config_file,
	    args.pytorch_dump_folder_path,
	    args.transfo_xl_dataset_file,
	)