






import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase	:		Dict										=   logging.get_logger(__name__)

__lowerCamelCase	:		List[str]										=   {
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

__lowerCamelCase	:		List[str]										=   {
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

__lowerCamelCase	:		int										=   {
    """vinai/phobert-base""": 2_56,
    """vinai/phobert-large""": 2_56,
}







def 			A__							(      _a  :					Tuple	):





 '''simple docstring'''







 snake_case__  :    Optional[int]							       =set()
 snake_case__  :    Optional[int]							       =word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)	)
  snake_case__  :    Any							       =char

 snake_case__  :    List[Any]							       =set(_a	)
 return pairs







class  _lowercase   (   _A						):
      _a							:						List[Any]										=							VOCAB_FILES_NAMES
      _a							:						Tuple										=							PRETRAINED_VOCAB_FILES_MAP
      _a							:						Union[str, Any]										=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      def __init__(     self				,			a				,			a				,			a="<s>"				,			a="</s>"				,			a="</s>"				,			a="<s>"				,			a="<unk>"				,			a="<pad>"				,			a="<mask>"				,			**a				,			):
       super().__init__(
           bos_token=a				,			eos_token=a				,			unk_token=a				,			sep_token=a				,			cls_token=a				,			pad_token=a				,			mask_token=a				,			**a				,			)

       snake_case__  :    Tuple							       =vocab_file
       snake_case__  :    int							       =merges_file

       snake_case__  :    Dict							       ={}
       snake_case__  :    List[str]							       =0
       snake_case__  :    Optional[int]							       =1
       snake_case__  :    Optional[int]							       =2
       snake_case__  :    List[str]							       =3

       self.add_from_file(a   )

       snake_case__  :    Optional[int]							       ={v: k for k, v in self.encoder.items()}

       with open(a				,			encoding="""utf-8"""   ) as merges_handle:
        snake_case__  :    int							       =merges_handle.read().split("""\n"""   )[:-1]
       snake_case__  :    Optional[Any]							       =[tuple(merge.split()[:-1]   ) for merge in merges]
       snake_case__  :    List[str]							       =dict(zip(a				,			range(len(a   )   )   )   )
       snake_case__  :    int							       ={}
      def 						lowercase__					(     self				,			a				,			a = None   ):

       if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       snake_case__  :    List[str]							       =[self.cls_token_id]
       snake_case__  :    Optional[int]							       =[self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep
      def 						lowercase__					(     self				,			a				,			a = None				,			a = False   ):

       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=a				,			token_ids_a=a				,			already_has_special_tokens=a   )

       if token_ids_a is None:
        return [1] + ([0] * len(a   )) + [1]
       return [1] + ([0] * len(a   )) + [1, 1] + ([0] * len(a   )) + [1]
      def 						lowercase__					(     self				,			a				,			a = None   ):
       snake_case__  :    Tuple							       =[self.sep_token_id]
       snake_case__  :    List[str]							       =[self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep   ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]
      @property
      def 						lowercase__					(     self   ):
       return len(self.encoder   )
      def 						lowercase__					(     self   ):
       return dict(self.encoder				,			**self.added_tokens_encoder   )
      def 						lowercase__					(     self				,			a   ):
       if token in self.cache:
        return self.cache[token]
       snake_case__  :    List[str]							       =tuple(a   )
       snake_case__  :    Dict							       =tuple(list(word[:-1]   ) + [word[-1] + """</w>"""]   )
       snake_case__  :    int							       =get_pairs(a   )

       if not pairs:
        return token

       while True:
        snake_case__  :    Any							       =min(a				,			key=lambda a   : self.bpe_ranks.get(a				,			float("""inf"""   )   )   )
        if bigram not in self.bpe_ranks:
         break
        snake_case__       ,   snake_case__  :    Optional[int]							       =bigram
        snake_case__  :    str							       =[]
        snake_case__  :    Union[str, Any]							       =0
        while i < len(a   ):
         try:
          snake_case__  :    List[str]							       =word.index(a				,			a   )
         except ValueError:
          new_word.extend(word[i:]   )
          break
         else:
          new_word.extend(word[i:j]   )
          snake_case__  :    Optional[Any]							       =j

         if word[i] == first and i < len(a   ) - 1 and word[i + 1] == second:
          new_word.append(first + second   )
          i += 2
         else:
          new_word.append(word[i]   )
          i += 1
        snake_case__  :    Optional[int]							       =tuple(a   )
        snake_case__  :    Dict							       =new_word
        if len(a   ) == 1:
         break
        else:
         snake_case__  :    List[Any]							       =get_pairs(a   )
       snake_case__  :    Optional[Any]							       ="""@@ """.join(a   )
       snake_case__  :    int							       =word[:-4]
       snake_case__  :    str							       =word
       return word
      def 						lowercase__					(     self				,			a   ):
       snake_case__  :    Union[str, Any]							       =[]

       snake_case__  :    Dict							       =re.findall(R"""\S+\n?"""				,			a   )

       for token in words:
        split_tokens.extend(list(self.bpe(a   ).split(""" """   )   )   )
       return split_tokens
      def 						lowercase__					(     self				,			a   ):
       return self.encoder.get(a				,			self.encoder.get(self.unk_token   )   )
      def 						lowercase__					(     self				,			a   ):
       return self.decoder.get(a				,			self.unk_token   )
      def 						lowercase__					(     self				,			a   ):
       snake_case__  :    Optional[int]							       =""" """.join(a   ).replace("""@@ """				,			""""""   ).strip()
       return out_string
      def 						lowercase__					(     self				,			a				,			a = None   ):
       if not os.path.isdir(a   ):
        logger.error(F"Vocabulary path ({save_directory}) should be a directory"   )
        return
       snake_case__  :    Any							       =os.path.join(
           a				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
       snake_case__  :    Dict							       =os.path.join(
           a				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

       if os.path.abspath(self.vocab_file   ) != os.path.abspath(a   ):
        copyfile(self.vocab_file				,			a   )

       if os.path.abspath(self.merges_file   ) != os.path.abspath(a   ):
        copyfile(self.merges_file				,			a   )

       return out_vocab_file, out_merge_file



      def 						lowercase__					(     self				,			a   ):
       if isinstance(a				,			a   ):
        try:
         with open(a				,			"""r"""				,			encoding="""utf-8"""   ) as fd:
          self.add_from_file(a   )
        except FileNotFoundError as fnfe:
         raise fnfe
        except UnicodeError:
         raise Exception(F"Incorrect encoding detected in {f}, please rebuild the dataset"   )
        return

       snake_case__  :    str							       =f.readlines()
       for lineTmp in lines:
        snake_case__  :    Any							       =lineTmp.strip()
        snake_case__  :    Tuple							       =line.rfind(""" """   )
        if idx == -1:
         raise ValueError("""Incorrect dictionary format, expected '<token> <cnt>'"""   )
        snake_case__  :    Dict							       =line[:idx]
        snake_case__  :    Optional[Any]							       =len(self.encoder   )













from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCamelCase	:		Dict										=   logging.get_logger(__name__)

__lowerCamelCase	:		Union[str, Any]										=   {
    """xlm-mlm-en-2048""": """https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json""",
    """xlm-mlm-ende-1024""": """https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-enfr-1024""": """https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json""",
    """xlm-mlm-enro-1024""": """https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json""",
    """xlm-mlm-tlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json""",
    """xlm-mlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json""",
    """xlm-clm-enfr-1024""": """https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json""",
    """xlm-clm-ende-1024""": """https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-17-1280""": """https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json""",
    """xlm-mlm-100-1280""": """https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json""",
}







class  _lowercase   (   _A						):
      _a							:						int										=							'xlm'
      _a							:						List[Any]										=							{
          'hidden_size': 'emb_dim',
          'num_attention_heads': 'n_heads',
          'num_hidden_layers': 'n_layers',
          'n_words': 'vocab_size',  # For backward compatibility
      }
      def __init__(     self				,			a=3_0_1_4_5				,			a=2_0_4_8				,			a=1_2				,			a=1_6				,			a=0.1				,			a=0.1				,			a=True				,			a=False				,			a=False				,			a=False				,			a=1				,			a=True				,			a=5_1_2				,			a=2_0_4_8**-0.5				,			a=1e-12				,			a=0.02				,			a=0				,			a=1				,			a=2				,			a=3				,			a=5				,			a=True				,			a="first"				,			a=True				,			a=None				,			a=True				,			a=0.1				,			a=5				,			a=5				,			a=0				,			a=0				,			a=2				,			a=0				,			**a				,			):
       snake_case__  :    str							       =vocab_size
       snake_case__  :    List[Any]							       =emb_dim
       snake_case__  :    List[str]							       =n_layers
       snake_case__  :    Any							       =n_heads
       snake_case__  :    Any							       =dropout
       snake_case__  :    Optional[int]							       =attention_dropout
       snake_case__  :    Dict							       =gelu_activation
       snake_case__  :    Tuple							       =sinusoidal_embeddings
       snake_case__  :    Optional[int]							       =causal
       snake_case__  :    Optional[int]							       =asm
       snake_case__  :    Optional[int]							       =n_langs
       snake_case__  :    Union[str, Any]							       =use_lang_emb
       snake_case__  :    Optional[Any]							       =layer_norm_eps
       snake_case__  :    Union[str, Any]							       =bos_index
       snake_case__  :    List[Any]							       =eos_index
       snake_case__  :    Optional[Any]							       =pad_index
       snake_case__  :    Union[str, Any]							       =unk_index
       snake_case__  :    Dict							       =mask_index
       snake_case__  :    str							       =is_encoder
       snake_case__  :    Optional[Any]							       =max_position_embeddings
       snake_case__  :    Any							       =embed_init_std
       snake_case__  :    Optional[Any]							       =init_std
       snake_case__  :    str							       =summary_type
       snake_case__  :    List[Any]							       =summary_use_proj
       snake_case__  :    str							       =summary_activation
       snake_case__  :    Optional[Any]							       =summary_proj_to_labels
       snake_case__  :    Optional[int]							       =summary_first_dropout
       snake_case__  :    Optional[Any]							       =start_n_top
       snake_case__  :    List[Any]							       =end_n_top
       snake_case__  :    Union[str, Any]							       =mask_token_id
       snake_case__  :    Any							       =lang_id

       if "n_words" in kwargs:
        snake_case__  :    int							       =kwargs["""n_words"""]

       super().__init__(pad_token_id=a				,			bos_token_id=a				,			**a   )







class  _lowercase   (   _A						):
      @property
      def 						lowercase__					(     self   ):
       if self.task == "multiple-choice":
        snake_case__  :    Any							       ={0: """batch""", 1: """choice""", 2: """sequence"""}
       else:
        snake_case__  :    Union[str, Any]							       ={0: """batch""", 1: """sequence"""}
       return OrderedDict(
           [
               ("""input_ids""", dynamic_axis),
               ("""attention_mask""", dynamic_axis),
               ("""token_type_ids""", dynamic_axis),
           ]   )





