



'''simple docstring'''



import argparse

import torch

from transformers import GPTaConfig, GPTaModel, load_tf_weights_in_gpta
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()



def   _A       (   _lowerCAmelCase  ,   _lowerCAmelCase  ,   _lowerCAmelCase					):



							"""simple docstring"""



							if gpta_config_file == "":
														__lowercase       	=GPTaConfig()
							else:
														__lowercase       	=GPTaConfig.from_json_file(_lowerCAmelCase					)
							__lowercase       	=GPTaModel(_lowerCAmelCase					)

							# Load weights from numpy
							load_tf_weights_in_gpta(_lowerCAmelCase  ,   _lowerCAmelCase  ,   _lowerCAmelCase					)

							# Save pytorch-model
							__lowercase       	=pytorch_dump_folder_path + '/' + WEIGHTS_NAME
							__lowercase       	=pytorch_dump_folder_path + '/' + CONFIG_NAME
							print(f"""Save PyTorch model to {pytorch_weights_dump_path}"""					)
							torch.save(model.state_dict()  ,   _lowerCAmelCase					)
							print(f"""Save configuration file to {pytorch_config_dump_path}"""					)
							with open(_lowerCAmelCase  ,   'w'  ,   encoding='utf-8'					) as f:
														f.write(config.to_json_string()					)


if __name__ == "__main__":
		lowerCamelCase	 =       argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    """--gpt2_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
		)
		parser.add_argument(
		    """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
		)
		parser.add_argument(
		    """--gpt2_config_file""",
		    default="""""",
		    type=str,
		    help=(
		        """An optional config json file corresponding to the pre-trained OpenAI model. \n"""
		        """This specifies the model architecture."""
		    ),
		)
		lowerCamelCase	 =       parser.parse_args()
		convert_gpta_checkpoint_to_pytorch(args.gpta_checkpoint_path, args.gpta_config_file, args.pytorch_dump_folder_path)








'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)


lowerCamelCase	 =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
lowerCamelCase	 =       {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

lowerCamelCase	 =       {
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    ["""input_ids""", """attention_mask"""]
				lowerCAmelCase__					    =    BartTokenizer

				def __init__(							self  :  Optional[int]  ,       _lowerCAmelCase  :  Any=None  ,       _lowerCAmelCase  :  Tuple=None  ,       _lowerCAmelCase  :  Optional[int]=None  ,       _lowerCAmelCase  :  str="replace"  ,       _lowerCAmelCase  :  List[Any]="<s>"  ,       _lowerCAmelCase  :  int="</s>"  ,       _lowerCAmelCase  :  Dict="</s>"  ,       _lowerCAmelCase  :  Optional[int]="<s>"  ,       _lowerCAmelCase  :  str="<unk>"  ,       _lowerCAmelCase  :  List[str]="<pad>"  ,       _lowerCAmelCase  :  Dict="<mask>"  ,       _lowerCAmelCase  :  Tuple=False  ,       _lowerCAmelCase  :  str=True  ,       **_lowerCAmelCase  :  Tuple  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       errors=_lowerCAmelCase  ,       bos_token=_lowerCAmelCase  ,       eos_token=_lowerCAmelCase  ,       sep_token=_lowerCAmelCase  ,       cls_token=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       pad_token=_lowerCAmelCase  ,       mask_token=_lowerCAmelCase  ,       add_prefix_space=_lowerCAmelCase  ,       trim_offsets=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											__lowercase       	=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																		__lowercase       	=getattr(_lowerCAmelCase  ,       pre_tok_state.pop('type'))
																		__lowercase       	=add_prefix_space
																		__lowercase       	=pre_tok_class(**_lowerCAmelCase)

											__lowercase       	=add_prefix_space

											# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
											__lowercase       	='post_processor'
											__lowercase       	=getattr(self.backend_tokenizer  ,       _lowerCAmelCase  ,       _lowerCAmelCase)
											if tokenizer_component_instance:
																		__lowercase       	=json.loads(tokenizer_component_instance.__getstate__())

																		# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																		if "sep" in state:
																									__lowercase       	=tuple(state['sep'])
																		if "cls" in state:
																									__lowercase       	=tuple(state['cls'])

																		__lowercase       	=False

																		if state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																									__lowercase       	=add_prefix_space
																									__lowercase       	=True

																		if state.get('trim_offsets'  ,       _lowerCAmelCase) != trim_offsets:
																									__lowercase       	=trim_offsets
																									__lowercase       	=True

																		if changes_to_apply:
																									__lowercase       	=getattr(_lowerCAmelCase  ,       state.pop('type'))
																									__lowercase       	=component_class(**_lowerCAmelCase)
																									setattr(self.backend_tokenizer  ,       _lowerCAmelCase  ,       _lowerCAmelCase)

				@property
				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											if self._mask_token is None:
																		if self.verbose:
																									logger.error('Using mask_token, but it is not set yet.')
																		return None
											return str(self._mask_token)

				@mask_token.setter
				def 				__lowerCamelCase     (							self  :  Union[str, Any]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											__lowercase       	=AddedToken(_lowerCAmelCase  ,       lstrip=_lowerCAmelCase  ,       rstrip=_lowerCAmelCase) if isinstance(_lowerCAmelCase  ,       _lowerCAmelCase) else value
											__lowercase       	=value

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       *_lowerCAmelCase  :  int  ,       **_lowerCAmelCase  :  str):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    'to use it with pretokenized inputs.')

											return super()._batch_encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       *_lowerCAmelCase  :  List[str]  ,       **_lowerCAmelCase  :  Tuple):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    'to use it with pretokenized inputs.')

											return super()._encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  int  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  List[str]=None):







											'''simple docstring'''

											__lowercase       	=[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																		return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

				def 				__lowerCamelCase     (							self  :  List[str]  ,       _lowerCAmelCase  :  List[int]  ,       _lowerCAmelCase  :  Optional[List[int]] = None):







											'''simple docstring'''

											__lowercase       	=[self.sep_token_id]
											__lowercase       	=[self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



