
import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
UpperCamelCase__      :		List[Any]       							=  data_utils.TransfoXLTokenizer
UpperCamelCase__      :		List[Any]       							=  data_utils.TransfoXLCorpus
UpperCamelCase__      :		str       							=  data_utils
UpperCamelCase__      :		Optional[int]       							=  data_utils
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""
   if transfo_xl_dataset_file:
      # Convert a pre-processed corpus (see original TensorFlow repo)
      with open(UpperCamelCase__    ,      """rb"""						) as fp:
         _SCREAMING_SNAKE_CASE      =				pickle.load(UpperCamelCase__    ,      encoding="""latin1"""						)
      # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
      _SCREAMING_SNAKE_CASE      =				pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
      print(F"Save vocabulary to {pytorch_vocab_dump_path}"						)
      _SCREAMING_SNAKE_CASE      =				corpus.vocab.__dict__
      torch.save(UpperCamelCase__    ,      UpperCamelCase__						)

      _SCREAMING_SNAKE_CASE      =				corpus.__dict__
      corpus_dict_no_vocab.pop("""vocab"""    ,      UpperCamelCase__						)
      _SCREAMING_SNAKE_CASE      =				pytorch_dump_folder_path + """/""" + CORPUS_NAME
      print(F"Save dataset to {pytorch_dataset_dump_path}"						)
      torch.save(UpperCamelCase__    ,      UpperCamelCase__						)

   if tf_checkpoint_path:
      # Convert a pre-trained TensorFlow model
      _SCREAMING_SNAKE_CASE      =				os.path.abspath(UpperCamelCase__						)
      _SCREAMING_SNAKE_CASE      =				os.path.abspath(UpperCamelCase__						)

      print(F"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."						)
      # Initialise PyTorch model
      if transfo_xl_config_file == "":
         _SCREAMING_SNAKE_CASE      =				TransfoXLConfig()
      else:
         _SCREAMING_SNAKE_CASE      =				TransfoXLConfig.from_json_file(UpperCamelCase__						)
      print(F"Building PyTorch model from configuration: {config}"						)
      _SCREAMING_SNAKE_CASE      =				TransfoXLLMHeadModel(UpperCamelCase__						)

      _SCREAMING_SNAKE_CASE      =				load_tf_weights_in_transfo_xl(UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__						)
      # Save pytorch-model
      _SCREAMING_SNAKE_CASE      =				os.path.join(UpperCamelCase__    ,      UpperCamelCase__						)
      _SCREAMING_SNAKE_CASE      =				os.path.join(UpperCamelCase__    ,      UpperCamelCase__						)
      print(F"Save PyTorch model to {os.path.abspath(UpperCamelCase__						)}"						)
      torch.save(model.state_dict()    ,      UpperCamelCase__						)
      print(F"Save configuration file to {os.path.abspath(UpperCamelCase__						)}"						)
      with open(UpperCamelCase__    ,      """w"""    ,      encoding="""utf-8"""						) as f:
         f.write(config.to_json_string()						)


if __name__ == "__main__":
     UpperCamelCase__      :		Dict       							=  argparse.ArgumentParser()
     parser.add_argument(
         "--pytorch_dump_folder_path",
         default=None,
         type=str,
         required=True,
         help="Path to the folder to store the PyTorch model or dataset/vocab.",
     )
     parser.add_argument(
         "--tf_checkpoint_path",
         default="",
         type=str,
         help="An optional path to a TensorFlow checkpoint path to be converted.",
     )
     parser.add_argument(
         "--transfo_xl_config_file",
         default="",
         type=str,
         help=(
             "An optional config json file corresponding to the pre-trained BERT model. \n"
             "This specifies the model architecture."
         ),
     )
     parser.add_argument(
         "--transfo_xl_dataset_file",
         default="",
         type=str,
         help="An optional dataset file to be converted in a vocabulary.",
     )
     UpperCamelCase__      :		int       							=  parser.parse_args()
     convert_transfo_xl_checkpoint_to_pytorch(
         args.tf_checkpoint_path,
         args.transfo_xl_config_file,
         args.pytorch_dump_folder_path,
         args.transfo_xl_dataset_file,
     )








'''simple docstring'''



def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     str:




   """simple docstring"""
   return "".join([hex(SCREAMING_SNAKE_CASE_						)[2:].zfill(2						).upper() for byte in list(SCREAMING_SNAKE_CASE_						)]						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     bytes:




   """simple docstring"""
   # Check data validity, following RFC3548
   # https://www.ietf.org/rfc/rfc3548.txt
   if (len(SCREAMING_SNAKE_CASE_						) % 2) != 0:
      raise ValueError(
          """Base16 encoded data is invalid:
Data does not have an even number of hex digits."""						)
   # Check the character set - the standard base16 alphabet
   # is uppercase according to RFC3548 section 6
   if not set(SCREAMING_SNAKE_CASE_						) <= set("""0123456789ABCDEF"""						):
      raise ValueError(
          """Base16 encoded data is invalid:
Data is not uppercase hex or it contains invalid characters."""						)
   # For every two hexadecimal digits (= a byte), turn it into an integer.
   # Then, string the result together into bytes, and return it.
   return bytes(int(data[i] + data[i + 1]    ,      16						) for i in range(0    ,      len(SCREAMING_SNAKE_CASE_						)    ,      2						)						)


if __name__ == "__main__":
     import doctest

     doctest.testmod()






