


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__lowerCamelCase         =			logging.get_logger(__name__)

__lowerCamelCase         =			{'''vocab_file''': '''spiece.model'''}

__lowerCamelCase         =			{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

__lowerCamelCase         =			{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
__lowerCamelCase         =			0
__lowerCamelCase         =			1
__lowerCamelCase         =			2
__lowerCamelCase         =			3
__lowerCamelCase         =			4


class   snake_case_ (__SCREAMING_SNAKE_CASE  ):




       """simple docstring"""






       _lowerCamelCase			    =				VOCAB_FILES_NAMES
       _lowerCamelCase			    =				PRETRAINED_VOCAB_FILES_MAP
       _lowerCamelCase			    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _lowerCamelCase			    =				'left'



       def __init__(     self     ,lowercase     ,lowercase=False     ,lowercase=True     ,lowercase=False     ,lowercase="<s>"     ,lowercase="</s>"     ,lowercase="<unk>"     ,lowercase="<sep>"     ,lowercase="<pad>"     ,lowercase="<cls>"     ,lowercase="<mask>"     ,lowercase=["<eop>", "<eod>"]     ,lowercase = None     ,**lowercase     ,):

             """simple docstring"""





             UpperCAmelCase_  :	Optional[int]				      =		AddedToken(_a     ,lstrip=_a     ,rstrip=_a) if isinstance(_a     ,_a) else mask_token

             UpperCAmelCase_  :	str				      =		{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=_a     ,remove_space=_a     ,keep_accents=_a     ,bos_token=_a     ,eos_token=_a     ,unk_token=_a     ,sep_token=_a     ,pad_token=_a     ,cls_token=_a     ,mask_token=_a     ,additional_special_tokens=_a     ,sp_model_kwargs=self.sp_model_kwargs     ,**_a     ,)

             UpperCAmelCase_  :	Union[str, Any]				      =		3

             UpperCAmelCase_  :	Optional[int]				      =		do_lower_case
             UpperCAmelCase_  :	Optional[int]				      =		remove_space
             UpperCAmelCase_  :	Optional[Any]				      =		keep_accents
             UpperCAmelCase_  :	Tuple				      =		vocab_file

             UpperCAmelCase_  :	int				      =		spm.SentencePieceProcessor(**self.sp_model_kwargs)
             self.sp_model.Load(_a)



       @property
       def  A_   (     self):

             """simple docstring"""





             return len(self.sp_model)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Union[str, Any]				      =		{self.convert_ids_to_tokens(_a): i for i in range(self.vocab_size)}
             vocab.update(self.added_tokens_encoder)
             return vocab



       def __getstate__(     self):

             """simple docstring"""





             UpperCAmelCase_  :	List[str]				      =		self.__dict__.copy()
             UpperCAmelCase_  :	Tuple				      =		None
             return state



       def __setstate__(     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	Union[str, Any]				      =		d

             # for backward compatibility
             if not hasattr(self     ,"sp_model_kwargs"):
                   UpperCAmelCase_  :	Optional[int]				      =		{}

             UpperCAmelCase_  :	Tuple				      =		spm.SentencePieceProcessor(**self.sp_model_kwargs)
             self.sp_model.Load(self.vocab_file)



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             if self.remove_space:
                   UpperCAmelCase_  :	str				      =		" ".join(inputs.strip().split())
             else:
                   UpperCAmelCase_  :	List[str]				      =		inputs
             UpperCAmelCase_  :	Tuple				      =		outputs.replace("``"     ,"\"").replace("\'\'"     ,"\"")

             if not self.keep_accents:
                   UpperCAmelCase_  :	List[Any]				      =		unicodedata.normalize("NFKD"     ,_a)
                   UpperCAmelCase_  :	List[Any]				      =		"".join([c for c in outputs if not unicodedata.combining(_a)])
             if self.do_lower_case:
                   UpperCAmelCase_  :	Tuple				      =		outputs.lower()

             return outputs



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	Optional[Any]				      =		self.preprocess_text(_a)
             UpperCAmelCase_  :	Tuple				      =		self.sp_model.encode(_a     ,out_type=_a)
             UpperCAmelCase_  :	int				      =		[]
             for piece in pieces:
                   if len(_a) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
                         UpperCAmelCase_  :	Optional[Any]				      =		self.sp_model.EncodeAsPieces(piece[:-1].replace(_a     ,""))
                         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                               if len(cur_pieces[0]) == 1:
                                     UpperCAmelCase_  :	Union[str, Any]				      =		cur_pieces[1:]
                               else:
                                     UpperCAmelCase_  :	Optional[Any]				      =		cur_pieces[0][1:]
                         cur_pieces.append(piece[-1])
                         new_pieces.extend(_a)
                   else:
                         new_pieces.append(_a)

             return new_pieces



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             return self.sp_model.PieceToId(_a)



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             return self.sp_model.IdToPiece(_a)



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	Tuple				      =		"".join(_a).replace(_a     ," ").strip()
             return out_string



       def  A_   (     self     ,lowercase     ,lowercase = False     ,lowercase = None     ,lowercase = True     ,**lowercase     ,):

             """simple docstring"""





             UpperCAmelCase_  :	List[Any]				      =		kwargs.pop("use_source_tokenizer"     ,_a)

             UpperCAmelCase_  :	Dict				      =		self.convert_ids_to_tokens(_a     ,skip_special_tokens=_a)

             # To avoid mixing byte-level and unicode for byte-level BPT
             # we need to build string separately for added tokens and byte-level tokens
             # cf. https://github.com/huggingface/transformers/issues/1133
             UpperCAmelCase_  :	Optional[Any]				      =		[]
             UpperCAmelCase_  :	Optional[int]				      =		[]
             for token in filtered_tokens:
                   if skip_special_tokens and token in self.all_special_ids:
                         continue
                   if token in self.added_tokens_encoder:
                         if current_sub_text:
                               sub_texts.append(self.convert_tokens_to_string(_a))
                               UpperCAmelCase_  :	Tuple				      =		[]
                         sub_texts.append(_a)
                   else:
                         current_sub_text.append(_a)
             if current_sub_text:
                   sub_texts.append(self.convert_tokens_to_string(_a))

             # Mimic the behavior of the Rust tokenizer:
             # By default, there are no spaces between special tokens
             UpperCAmelCase_  :	Union[str, Any]				      =		"".join(_a)

             UpperCAmelCase_  :	Any				      =		(
                 clean_up_tokenization_spaces
                 if clean_up_tokenization_spaces is not None
                 else self.clean_up_tokenization_spaces
             )
             if clean_up_tokenization_spaces:
                   UpperCAmelCase_  :	int				      =		self.clean_up_tokenization(_a)
                   return clean_text
             else:
                   return text



       def  A_   (     self     ,lowercase     ,lowercase = None):

             """simple docstring"""





             UpperCAmelCase_  :	List[str]				      =		[self.sep_token_id]
             UpperCAmelCase_  :	Optional[int]				      =		[self.cls_token_id]
             if token_ids_a is None:
                   return token_ids_a + sep + cls
             return token_ids_a + sep + token_ids_a + sep + cls



       def  A_   (     self     ,lowercase     ,lowercase = None     ,lowercase = False):

             """simple docstring"""





             if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=_a     ,token_ids_a=_a     ,already_has_special_tokens=_a)

             if token_ids_a is not None:
                   return ([0] * len(_a)) + [1] + ([0] * len(_a)) + [1, 1]
             return ([0] * len(_a)) + [1, 1]



       def  A_   (     self     ,lowercase     ,lowercase = None):

             """simple docstring"""





             UpperCAmelCase_  :	Any				      =		[self.sep_token_id]
             UpperCAmelCase_  :	Dict				      =		[2]

             if token_ids_a is None:
                   return len(token_ids_a + sep) * [0] + cls_segment_id
             return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id



       def  A_   (     self     ,lowercase     ,lowercase = None):

             """simple docstring"""





             if not os.path.isdir(_a):
                   logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
                   return
             UpperCAmelCase_  :	int				      =		os.path.join(
                 _a     ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

             if os.path.abspath(self.vocab_file) != os.path.abspath(_a) and os.path.isfile(self.vocab_file):
                   copyfile(self.vocab_file     ,_a)
             elif not os.path.isfile(self.vocab_file):
                   with open(_a     ,"wb") as fi:
                         UpperCAmelCase_  :	List[Any]				      =		self.sp_model.serialized_model_proto()
                         fi.write(_a)

             return (out_vocab_file,)





import argparse

import torch

from transformers import FunnelBaseModel, FunnelConfig, FunnelModel, load_tf_weights_in_funnel
from transformers.utils import logging


logging.set_verbosity_info()







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     Optional[Any]:






      '''simple docstring'''







      UpperCAmelCase_  :	List[str]				      =		FunnelConfig.from_json_file(__snake_case    )
      print(F"""Building PyTorch model from configuration: {config}"""    )
      UpperCAmelCase_  :	str				      =		FunnelBaseModel(__snake_case    ) if base_model else FunnelModel(__snake_case    )

      # Load weights from tf checkpoint
      load_tf_weights_in_funnel(__snake_case					,  __snake_case					,  __snake_case    )

      # Save pytorch-model
      print(F"""Save PyTorch model to {pytorch_dump_path}"""    )
      torch.save(model.state_dict()					,  __snake_case    )


if __name__ == "__main__":
   __lowerCamelCase         =			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
   )
   parser.add_argument(
       '''--config_file''',
       default=None,
       type=str,
       required=True,
       help='''The config json file corresponding to the pre-trained model. \nThis specifies the model architecture.''',
   )
   parser.add_argument(
       '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
   )
   parser.add_argument(
       '''--base_model''', action='''store_true''', help='''Whether you want just the base model (no decoder) or not.'''
   )
   __lowerCamelCase         =			parser.parse_args()
   convert_tf_checkpoint_to_pytorch(
       args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path, args.base_model
   )
