


import dataclasses
import re
from dataclasses import dataclass
from functools import total_ordering
from typing import Optional, Union


__UpperCamelCase    :						Tuple     =							re.compile(R'^(?P<major>\d+)' R'\.(?P<minor>\d+)' R'\.(?P<patch>\d+)$')



@total_ordering
@dataclass
class 							_UpperCamelCase :

   '''simple docstring'''







   a_				:							Tuple    =      42
   a_				:							List[Any]    =      None
   a_				:							Dict    =      None
   a_				:							Optional[int]    =      None
   a_				:							Dict    =      None

   def   _snake_case    (  self  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase     :      Dict       						=						_str_to_version_tuple(self.version_str							)

   def __repr__(  self  : Optional[int]							):






         '''simple docstring'''



         return F"""{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}"""

   @property
   def   _snake_case    (  self  : int							):






         '''simple docstring'''



         return self.major, self.minor, self.patch

   def   _snake_case    (  self  : Union[str, Any]       ,  _lowerCamelCase  : Union[str, Any]							):






         '''simple docstring'''



         if isinstance(_lowercase       ,  _lowercase							):
               return Version(_lowercase							)
         elif isinstance(_lowercase       ,  _lowercase							):
               return other
         raise TypeError(F"""{other} (type {type(_lowercase							)}) cannot be compared to version."""							)

   def __eq__(  self  : List[str]       ,  _lowerCamelCase  : int							):






         '''simple docstring'''



         try:
               __lowerCamelCase     :      str       						=						self._validate_operand(_lowercase							)
         except (TypeError, ValueError):
               return False
         else:
               return self.tuple == other.tuple

   def __lt__(  self  : Optional[int]       ,  _lowerCamelCase  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						self._validate_operand(_lowercase							)
         return self.tuple < other.tuple

   def __hash__(  self  : Optional[int]							):






         '''simple docstring'''



         return hash(_version_tuple_to_str(self.tuple							)							)

   @classmethod
   def   _snake_case    (  cls  : List[str]       ,  _lowerCamelCase  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      Union[str, Any]       						=						{f.name for f in dataclasses.fields(cls							)}
         return cls(**{k: v for k, v in dic.items() if k in field_names}							)


   def   _snake_case    (  self  : str							):






         '''simple docstring'''



         return self.version_str





def   _UpperCAmelCase       (     UpperCAmelCase      :			Dict      ):
      """simple docstring"""




      __lowerCamelCase     :      List[str]       						=						_VERSION_REG.match(UpperCamelCase__      )
      if not res:
            raise ValueError(f"""Invalid version \'{version_str}\'. Format should be x.y.z with {{x,y,z}} being digits."""      )
      return tuple(int(UpperCamelCase__      ) for v in [res.group("""major"""      ), res.group("""minor"""      ), res.group("""patch"""      )]      )





def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[int]      ):
      """simple docstring"""




      return ".".join(str(UpperCamelCase__      ) for v in version_tuple      )







import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing



class 							_UpperCamelCase (						A						):

   '''simple docstring'''







   def __init__(  self  : str       ,  _lowerCamelCase  : str = "▁"       ,  _lowerCamelCase  : bool = True       ,  _lowerCamelCase  : Union[str, AddedToken] = "<unk>"       ,  _lowerCamelCase  : Union[str, AddedToken] = "</s>"       ,  _lowerCamelCase  : Union[str, AddedToken] = "<pad>"       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Tuple       						=						{
             """pad""": {"""id""": 0, """token""": pad_token},
             """eos""": {"""id""": 1, """token""": eos_token},
             """unk""": {"""id""": 2, """token""": unk_token},
         }

         __lowerCamelCase     :      Optional[Any]       						=						[None] * len(self.special_tokens							)
         for token_dict in self.special_tokens.values():
               __lowerCamelCase     :      Tuple       						=						token_dict["""token"""]

         __lowerCamelCase     :      Optional[int]       						=						Tokenizer(Unigram()							)

         __lowerCamelCase     :      str       						=						normalizers.Sequence(
             [
                 normalizers.Nmt(),
                 normalizers.NFKC(),
                 normalizers.Replace(Regex(""" {2,}"""							)       ,  """ """							),
                 normalizers.Lowercase(),
             ]							)
         __lowerCamelCase     :      int       						=						pre_tokenizers.Sequence(
             [
                 pre_tokenizers.Metaspace(replacement=_lowerCamelCase       ,  add_prefix_space=_lowerCamelCase							),
                 pre_tokenizers.Digits(individual_digits=_lowerCamelCase							),
                 pre_tokenizers.Punctuation(),
             ]							)
         __lowerCamelCase     :      Tuple       						=						decoders.Metaspace(replacement=_lowerCamelCase       ,  add_prefix_space=_lowerCamelCase							)

         __lowerCamelCase     :      Tuple       						=						TemplateProcessing(
             single=F"""$A {self.special_tokens["eos"]["token"]}"""       ,  special_tokens=[(self.special_tokens["""eos"""]["""token"""], self.special_tokens["""eos"""]["""id"""])]       ,  )

         __lowerCamelCase     :      Tuple       						=						{
             """model""": """SentencePieceUnigram""",
             """replacement""": replacement,
             """add_prefix_space""": add_prefix_space,
         }

         super().__init__(_lowerCamelCase       ,  _lowerCamelCase							)

   def   _snake_case    (  self  : Dict       ,  _lowerCamelCase  : Union[str, List[str]]       ,  _lowerCamelCase  : int = 8_0_0_0       ,  _lowerCamelCase  : bool = True       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						trainers.UnigramTrainer(
             vocab_size=_lowerCamelCase       ,  special_tokens=self.special_tokens_list       ,  show_progress=_lowerCamelCase       ,  )

         if isinstance(_lowerCamelCase       ,  _lowerCamelCase							):
               __lowerCamelCase     :      Optional[int]       						=						[files]
         self._tokenizer.train(_lowerCamelCase       ,  trainer=_lowerCamelCase							)

         self.add_unk_id()

   def   _snake_case    (  self  : List[Any]       ,  _lowerCamelCase  : Union[Iterator[str], Iterator[Iterator[str]]]       ,  _lowerCamelCase  : int = 8_0_0_0       ,  _lowerCamelCase  : bool = True       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						trainers.UnigramTrainer(
             vocab_size=_lowerCamelCase       ,  special_tokens=self.special_tokens_list       ,  show_progress=_lowerCamelCase       ,  )

         self._tokenizer.train_from_iterator(_lowerCamelCase       ,  trainer=_lowerCamelCase							)

         self.add_unk_id()


   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      List[str]       						=						json.loads(self._tokenizer.to_str()							)

         __lowerCamelCase     :      Optional[int]       						=						self.special_tokens["""unk"""]["""id"""]

         __lowerCamelCase     :      List[str]       						=						Tokenizer.from_str(json.dumps(_lowerCamelCase							)							)



