
"""simple docstring"""





import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				""
			snake_case	      =				(
			    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
			)
			snake_case	      =				None  # compression type in fsspec. ex: "gzip"
			snake_case	      =				None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz




			def __init__(				self						:					List[str]       ,   _snake_case						:					str = ""       ,   _snake_case						:					Optional[str] = None       ,   _snake_case						:					Optional[dict] = None       ,   **_snake_case						:					Union[str, Any]	)      -> int:







								"""simple docstring"""
								super().__init__(self       ,   **_snake_case	)
								# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
								A_				 =						fsspec.open(
								    _snake_case       ,   mode="rb"       ,   protocol=_snake_case       ,   compression=self.compression       ,   client_kwargs={
								        "requote_redirect_url": False,  # see https://github.com/huggingface/datasets/pull/5459
								        "trust_env": True,  # Enable reading proxy env variables.
								        **(target_options or {}).pop("client_kwargs"       ,   {}	),  # To avoid issues if it was already passed.
								    }       ,   **(target_options or {})       ,   )
								A_				 =						os.path.basename(self.file.path.split("::"	)[0]	)
								A_				 =						(
								    self.compressed_name[: self.compressed_name.rindex("."	)]
								    if "." in self.compressed_name
								    else self.compressed_name
								)
								A_				 =						None




			@classmethod
			def      lowerCamelCase__     (				cls						:					str       ,   _snake_case						:					int	)      -> Tuple:







								"""simple docstring"""
								# compressed file paths are always relative to the archive root
								return super()._strip_protocol(_snake_case	).lstrip("/"	)




			def      lowerCamelCase__     (				self						:					int	)      -> List[Any]:







								"""simple docstring"""
								if self.dir_cache is None:
													A_				 =						{**self.file.fs.info(self.file.path	), "name": self.uncompressed_name}
													A_				 =						{f["name"]: f}




			def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					str	)      -> str:







								"""simple docstring"""
								return self.file.open().read()






			def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					str       ,   _snake_case						:					str = "rb"       ,   _snake_case						:					Any=None       ,   _snake_case						:					Any=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					List[Any]       ,   )      -> Any:







								"""simple docstring"""
								A_				 =						self._strip_protocol(_snake_case	)
								if mode != "rb":
													raise ValueError(F'Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\''	)
								return self.file.open()





class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				"bz2"
			snake_case	      =				"bz2"
			snake_case	      =				".bz2"





class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				"gzip"
			snake_case	      =				"gzip"
			snake_case	      =				".gz"





class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				"lz4"
			snake_case	      =				"lz4"
			snake_case	      =				".lz4"





class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				"xz"
			snake_case	      =				"xz"
			snake_case	      =				".xz"





class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				"zstd"
			snake_case	      =				"zstd"
			snake_case	      =				".zst"




			def __init__(				self						:					Tuple       ,   _snake_case						:					str       ,   _snake_case						:					str = "rb"       ,   _snake_case						:					Optional[str] = None       ,   _snake_case						:					Optional[dict] = None       ,   _snake_case						:					int = DEFAULT_BLOCK_SIZE       ,   **_snake_case						:					Any       ,   )      -> Optional[int]:







								"""simple docstring"""
								super().__init__(
								    fo=_snake_case       ,   mode=_snake_case       ,   target_protocol=_snake_case       ,   target_options=_snake_case       ,   block_size=_snake_case       ,   **_snake_case       ,   )
								# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
								#
								# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
								#     out.close = close
								# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
								#
								# see https://github.com/intake/filesystem_spec/issues/725
								A_				 =						self.file.__enter__



								class __lowerCAmelCase				:




											"""simple docstring"""







											def __init__(				self						:					int       ,   _snake_case						:					Any	)      -> List[str]:







																"""simple docstring"""
																A_				 =						file_




											def __enter__(				self						:					Tuple	)      -> str:







																"""simple docstring"""
																self._file.__enter__()
																return self




											def __exit__(				self						:					List[Any]       ,   *_snake_case						:					Optional[Any]       ,   **_snake_case						:					int	)      -> Tuple:







																"""simple docstring"""
																self._file.__exit__(*_snake_case       ,   **_snake_case	)




											def __iter__(				self						:					str	)      -> Union[str, Any]:







																"""simple docstring"""
																return iter(self._file	)




											def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> Any:







																"""simple docstring"""
																return next(self._file	)






											def __getattr__(				self						:					Any       ,   _snake_case						:					Any	)      -> List[str]:







																"""simple docstring"""
																return getattr(self._file       ,   _snake_case	)

								def fixed_enter(*_snake_case						:					int       ,   **_snake_case						:					Dict	):
													return WrappedFile(_enter(*_snake_case       ,   **_snake_case	)	)

								A_				 =						fixed_enter






"""simple docstring"""





from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand
def 							A_     (__a				):

					'''simple docstring'''





					return ConvertCommand(
					    args.model_type				,   args.tf_checkpoint				,   args.pytorch_dump_output				,   args.config				,   args.finetuning_task_name				)


UpperCamelCase_					:						Tuple				= '''
transformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires
TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.
'''




class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			@staticmethod
			def      lowerCamelCase__     (				_snake_case						:					ArgumentParser	)      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						parser.add_parser(
								    "convert"       ,   help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."       ,   )
								train_parser.add_argument("--model_type"       ,   type=_snake_case       ,   required=_snake_case       ,   help="Model's type."	)
								train_parser.add_argument(
								    "--tf_checkpoint"       ,   type=_snake_case       ,   required=_snake_case       ,   help="TensorFlow checkpoint path or folder."	)
								train_parser.add_argument(
								    "--pytorch_dump_output"       ,   type=_snake_case       ,   required=_snake_case       ,   help="Path to the PyTorch saved model output."	)
								train_parser.add_argument("--config"       ,   type=_snake_case       ,   default=""       ,   help="Configuration file path or folder."	)
								train_parser.add_argument(
								    "--finetuning_task_name"       ,   type=_snake_case       ,   default=_snake_case       ,   help="Optional fine-tuning task name if the TF model was a finetuned model."       ,   )
								train_parser.set_defaults(func=_snake_case	)




			def __init__(				self						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   *_snake_case						:					Optional[int]       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						logging.get_logger("transformers-cli/converting"	)

								self._logger.info(F'Loading model {model_type}'	)
								A_				 =						model_type
								A_				 =						tf_checkpoint
								A_				 =						pytorch_dump_output
								A_				 =						config
								A_				 =						finetuning_task_name






			def      lowerCamelCase__     (				self						:					Tuple	)      -> Dict:







								"""simple docstring"""
								if self._model_type == "albert":
													try:
																		from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
																		    convert_tf_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "bert":
													try:
																		from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
																		    convert_tf_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "funnel":
													try:
																		from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
																		    convert_tf_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "t5":
													try:
																		from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "gpt":
													from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
													    convert_openai_checkpoint_to_pytorch,
													)

													convert_openai_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "transfo_xl":
													try:
																		from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
																		    convert_transfo_xl_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													if "ckpt" in self._tf_checkpoint.lower():
																		A_				 =						self._tf_checkpoint
																		A_				 =						""
													else:
																		A_				 =						self._tf_checkpoint
																		A_				 =						""
													convert_transfo_xl_checkpoint_to_pytorch(
													    _snake_case       ,   self._config       ,   self._pytorch_dump_output       ,   _snake_case	)
								elif self._model_type == "gpt2":
													try:
																		from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
																		    convert_gpta_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "xlnet":
													try:
																		from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
																		    convert_xlnet_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_xlnet_checkpoint_to_pytorch(
													    self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output       ,   self._finetuning_task_name	)
								elif self._model_type == "xlm":
													from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
													    convert_xlm_checkpoint_to_pytorch,
													)

													convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._pytorch_dump_output	)
								elif self._model_type == "lxmert":
													from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
													    convert_lxmert_checkpoint_to_pytorch,
													)

													convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._pytorch_dump_output	)
								elif self._model_type == "rembert":
													from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
													    convert_rembert_tf_checkpoint_to_pytorch,
													)

													convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								else:
													raise ValueError(
													    "--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]"	)




