






'''simple docstring'''



import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
SCREAMING_SNAKE_CASE						=	sys.version_info >= (3, 10)


def 							lowercase_				(    __A :       Tuple=None					,      __A :       List[str]=None			)       ->      Union[str, Any]:



 """simple docstring"""




 return field(default_factory=lambda: default					,      metadata=__A			)
@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			42
      UpperCamelCase_								=			42
      UpperCamelCase_								=			42
      UpperCamelCase_								=			42





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			42
      UpperCamelCase_								=			field(default='''toto'''				,       metadata={'''help''': '''help message'''}				)





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			False
      UpperCamelCase_								=			True
      UpperCamelCase_								=			None





class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			'''titi'''
      UpperCamelCase_								=			'''toto'''





class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			'''titi'''
      UpperCamelCase_								=			'''toto'''
      UpperCamelCase_								=			42





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			"toto"






      def 				A__      ( self			:		int		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   List[str]				  =BasicEnum(self.foo		)





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			"toto"






      def 				A__      ( self			:		List[Any]		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =MixedTypeEnum(self.foo		)





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			None
      UpperCamelCase_								=			field(default=__A				,       metadata={'''help''': '''help message'''}				)
      UpperCamelCase_								=			None
      UpperCamelCase_								=			list_field(default=[]				)
      UpperCamelCase_								=			list_field(default=[]				)





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			list_field(default=[]				)
      UpperCamelCase_								=			list_field(default=[1, 2, 3]				)
      UpperCamelCase_								=			list_field(default=['''Hallo''', '''Bonjour''', '''Hello''']				)
      UpperCamelCase_								=			list_field(default=[0.1, 0.2, 0.3]				)





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			field()
      UpperCamelCase_								=			field()
      UpperCamelCase_								=			field()






      def 				A__      ( self			:		Tuple		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   List[Any]				  =BasicEnum(self.required_enum		)





@dataclass
class UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			42
      UpperCamelCase_								=			field()
      UpperCamelCase_								=			None
      UpperCamelCase_								=			field(default='''toto'''				,       metadata={'''help''': '''help message'''}				)
      UpperCamelCase_								=			list_field(default=['''Hallo''', '''Bonjour''', '''Hello''']				)


if is_python_no_less_than_3_10:
      @dataclass
      class UpperCAmelCase_					:




            """simple docstring"""




            UpperCamelCase_								=			False
            UpperCamelCase_								=			True
            UpperCamelCase_								=			None





      @dataclass
      class UpperCAmelCase_					:




            """simple docstring"""




            UpperCamelCase_								=			None
            UpperCamelCase_								=			field(default=__A				,       metadata={'''help''': '''help message'''}				)
            UpperCamelCase_								=			None
            UpperCamelCase_								=			list_field(default=[]				)
            UpperCamelCase_								=			list_field(default=[]				)





class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      def 				A__      ( self			:		List[Any]     ,      UpperCAmelCase			:		argparse.ArgumentParser     ,      UpperCAmelCase			:		argparse.ArgumentParser		)  ->  Any:


       '''simple docstring'''

       self.assertEqual(len(a._actions		)     ,      len(b._actions		)		)
       for x, y in zip(a._actions     ,      b._actions		):
        lowercase					:   Optional[Any]				  ={k: v for k, v in vars(UpperCAmelCase		).items() if k != '''container'''}
        lowercase					:   Union[str, Any]				  ={k: v for k, v in vars(UpperCAmelCase		).items() if k != '''container'''}

        # Choices with mixed type have custom function as "type"
        # So we need to compare results directly for equality
        if xx.get('''choices'''     ,      UpperCAmelCase		) and yy.get('''choices'''     ,      UpperCAmelCase		):
         for expected_choice in yy["choices"] + xx["choices"]:
          self.assertEqual(xx['''type'''](UpperCAmelCase		)     ,      yy['''type'''](UpperCAmelCase		)		)
         del xx["type"], yy["type"]

        self.assertEqual(UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		str		)  ->  Any:


       '''simple docstring'''

       lowercase					:   List[str]				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Any				  =argparse.ArgumentParser()
       expected.add_argument('''--foo'''     ,      type=UpperCAmelCase     ,      required=UpperCAmelCase		)
       expected.add_argument('''--bar'''     ,      type=UpperCAmelCase     ,      required=UpperCAmelCase		)
       expected.add_argument('''--baz'''     ,      type=UpperCAmelCase     ,      required=UpperCAmelCase		)
       expected.add_argument('''--flag'''     ,      type=UpperCAmelCase     ,      default=UpperCAmelCase     ,      const=UpperCAmelCase     ,      nargs='''?'''		)
       self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)

       lowercase					:   Optional[Any]				  =['''--foo''', '''1''', '''--baz''', '''quux''', '''--bar''', '''0.5''']
       ((lowercase)  ,		)					:   str				  =parser.parse_args_into_dataclasses(UpperCAmelCase     ,      look_for_args_file=UpperCAmelCase		)
       self.assertFalse(example.flag		)






      def 				A__      ( self			:		int		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   str				  =argparse.ArgumentParser()
       expected.add_argument('''--foo'''     ,      default=42     ,      type=UpperCAmelCase		)
       expected.add_argument('''--baz'''     ,      default='''toto'''     ,      type=UpperCAmelCase     ,      help='''help message'''		)
       self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		Union[str, Any]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   Dict				  =argparse.ArgumentParser()
       expected.add_argument('''--foo'''     ,      type=UpperCAmelCase     ,      default=UpperCAmelCase     ,      const=UpperCAmelCase     ,      nargs='''?'''		)
       expected.add_argument('''--baz'''     ,      type=UpperCAmelCase     ,      default=UpperCAmelCase     ,      const=UpperCAmelCase     ,      nargs='''?'''		)
       # A boolean no_* argument always has to come after its "default: True" regular counter-part
       # and its default must be set to False
       expected.add_argument('''--no_baz'''     ,      action='''store_false'''     ,      default=UpperCAmelCase     ,      dest='''baz'''		)
       expected.add_argument('''--opt'''     ,      type=UpperCAmelCase     ,      default=UpperCAmelCase		)

       lowercase					:   Optional[int]				  =[WithDefaultBoolExample]
       if is_python_no_less_than_3_10:
        dataclass_types.append(UpperCAmelCase		)

       for dataclass_type in dataclass_types:
        lowercase					:   Any				  =HfArgumentParser(UpperCAmelCase		)
        self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)

        lowercase					:   Any				  =parser.parse_args([]		)
        self.assertEqual(UpperCAmelCase     ,      Namespace(foo=UpperCAmelCase     ,      baz=UpperCAmelCase     ,      opt=UpperCAmelCase		)		)

        lowercase					:   Optional[Any]				  =parser.parse_args(['''--foo''', '''--no_baz''']		)
        self.assertEqual(UpperCAmelCase     ,      Namespace(foo=UpperCAmelCase     ,      baz=UpperCAmelCase     ,      opt=UpperCAmelCase		)		)

        lowercase					:   Optional[Any]				  =parser.parse_args(['''--foo''', '''--baz''']		)
        self.assertEqual(UpperCAmelCase     ,      Namespace(foo=UpperCAmelCase     ,      baz=UpperCAmelCase     ,      opt=UpperCAmelCase		)		)

        lowercase					:   int				  =parser.parse_args(['''--foo''', '''True''', '''--baz''', '''True''', '''--opt''', '''True''']		)
        self.assertEqual(UpperCAmelCase     ,      Namespace(foo=UpperCAmelCase     ,      baz=UpperCAmelCase     ,      opt=UpperCAmelCase		)		)

        lowercase					:   Optional[int]				  =parser.parse_args(['''--foo''', '''False''', '''--baz''', '''False''', '''--opt''', '''False''']		)
        self.assertEqual(UpperCAmelCase     ,      Namespace(foo=UpperCAmelCase     ,      baz=UpperCAmelCase     ,      opt=UpperCAmelCase		)		)






      def 				A__      ( self			:		List[str]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Optional[Any]				  =argparse.ArgumentParser()
       expected.add_argument(
           '''--foo'''     ,      default='''toto'''     ,      choices=['''titi''', '''toto''', 42]     ,      type=make_choice_type_function(['''titi''', '''toto''', 42]		)     ,      )
       self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)

       lowercase					:   Union[str, Any]				  =parser.parse_args([]		)
       self.assertEqual(args.foo     ,      '''toto'''		)
       lowercase					:   Dict				  =parser.parse_args_into_dataclasses([]		)[0]
       self.assertEqual(enum_ex.foo     ,      MixedTypeEnum.toto		)

       lowercase					:   Union[str, Any]				  =parser.parse_args(['''--foo''', '''titi''']		)
       self.assertEqual(args.foo     ,      '''titi'''		)
       lowercase					:   Union[str, Any]				  =parser.parse_args_into_dataclasses(['''--foo''', '''titi''']		)[0]
       self.assertEqual(enum_ex.foo     ,      MixedTypeEnum.titi		)

       lowercase					:   int				  =parser.parse_args(['''--foo''', '''42''']		)
       self.assertEqual(args.foo     ,      42		)
       lowercase					:   List[Any]				  =parser.parse_args_into_dataclasses(['''--foo''', '''42''']		)[0]
       self.assertEqual(enum_ex.foo     ,      MixedTypeEnum.fourtytwo		)






      def 				A__      ( self			:		Optional[int]		)  ->  int:


       '''simple docstring'''

       @dataclass
       class UpperCAmelCase_					:




             """simple docstring"""




             UpperCamelCase_								=			"toto"





       lowercase					:   int				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Optional[Any]				  =argparse.ArgumentParser()
       expected.add_argument(
           '''--foo'''     ,      default='''toto'''     ,      choices=('''titi''', '''toto''', 42)     ,      type=make_choice_type_function(['''titi''', '''toto''', 42]		)     ,      )
       self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)

       lowercase					:   str				  =parser.parse_args([]		)
       self.assertEqual(args.foo     ,      '''toto'''		)

       lowercase					:   Tuple				  =parser.parse_args(['''--foo''', '''titi''']		)
       self.assertEqual(args.foo     ,      '''titi'''		)

       lowercase					:   str				  =parser.parse_args(['''--foo''', '''42''']		)
       self.assertEqual(args.foo     ,      42		)






      def 				A__      ( self			:		Optional[int]		)  ->  Any:


       '''simple docstring'''

       lowercase					:   str				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   List[str]				  =argparse.ArgumentParser()
       expected.add_argument('''--foo_int'''     ,      nargs='''+'''     ,      default=[]     ,      type=UpperCAmelCase		)
       expected.add_argument('''--bar_int'''     ,      nargs='''+'''     ,      default=[1, 2, 3]     ,      type=UpperCAmelCase		)
       expected.add_argument('''--foo_str'''     ,      nargs='''+'''     ,      default=['''Hallo''', '''Bonjour''', '''Hello''']     ,      type=UpperCAmelCase		)
       expected.add_argument('''--foo_float'''     ,      nargs='''+'''     ,      default=[0.1, 0.2, 0.3]     ,      type=UpperCAmelCase		)

       self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)

       lowercase					:   Optional[Any]				  =parser.parse_args([]		)
       self.assertEqual(
           UpperCAmelCase     ,      Namespace(foo_int=[]     ,      bar_int=[1, 2, 3]     ,      foo_str=['''Hallo''', '''Bonjour''', '''Hello''']     ,      foo_float=[0.1, 0.2, 0.3]		)     ,      )

       lowercase					:   List[str]				  =parser.parse_args('''--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7'''.split()		)
       self.assertEqual(UpperCAmelCase     ,      Namespace(foo_int=[1]     ,      bar_int=[2, 3]     ,      foo_str=['''a''', '''b''', '''c''']     ,      foo_float=[0.1, 0.7]		)		)






      def 				A__      ( self			:		str		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   List[Any]				  =argparse.ArgumentParser()
       expected.add_argument('''--foo'''     ,      default=UpperCAmelCase     ,      type=UpperCAmelCase		)
       expected.add_argument('''--bar'''     ,      default=UpperCAmelCase     ,      type=UpperCAmelCase     ,      help='''help message'''		)
       expected.add_argument('''--baz'''     ,      default=UpperCAmelCase     ,      type=UpperCAmelCase		)
       expected.add_argument('''--ces'''     ,      nargs='''+'''     ,      default=[]     ,      type=UpperCAmelCase		)
       expected.add_argument('''--des'''     ,      nargs='''+'''     ,      default=[]     ,      type=UpperCAmelCase		)

       lowercase					:   int				  =[OptionalExample]
       if is_python_no_less_than_3_10:
        dataclass_types.append(UpperCAmelCase		)

       for dataclass_type in dataclass_types:
        lowercase					:   Union[str, Any]				  =HfArgumentParser(UpperCAmelCase		)

        self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)

        lowercase					:   Any				  =parser.parse_args([]		)
        self.assertEqual(UpperCAmelCase     ,      Namespace(foo=UpperCAmelCase     ,      bar=UpperCAmelCase     ,      baz=UpperCAmelCase     ,      ces=[]     ,      des=[]		)		)

        lowercase					:   Tuple				  =parser.parse_args('''--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3'''.split()		)
        self.assertEqual(UpperCAmelCase     ,      Namespace(foo=12     ,      bar=3.1_4     ,      baz='''42'''     ,      ces=['''a''', '''b''', '''c''']     ,      des=[1, 2, 3]		)		)






      def 				A__      ( self			:		Union[str, Any]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   str				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Dict				  =argparse.ArgumentParser()
       expected.add_argument('''--required_list'''     ,      nargs='''+'''     ,      type=UpperCAmelCase     ,      required=UpperCAmelCase		)
       expected.add_argument('''--required_str'''     ,      type=UpperCAmelCase     ,      required=UpperCAmelCase		)
       expected.add_argument(
           '''--required_enum'''     ,      type=make_choice_type_function(['''titi''', '''toto''']		)     ,      choices=['''titi''', '''toto''']     ,      required=UpperCAmelCase     ,      )
       self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		Union[str, Any]		)  ->  int:


       '''simple docstring'''

       lowercase					:   Optional[int]				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Union[str, Any]				  =argparse.ArgumentParser()
       expected.add_argument('''--foo'''     ,      type=UpperCAmelCase     ,      required=UpperCAmelCase		)
       expected.add_argument(
           '''--required_enum'''     ,      type=make_choice_type_function(['''titi''', '''toto''']		)     ,      choices=['''titi''', '''toto''']     ,      required=UpperCAmelCase     ,      )
       expected.add_argument('''--opt'''     ,      type=UpperCAmelCase     ,      default=UpperCAmelCase		)
       expected.add_argument('''--baz'''     ,      default='''toto'''     ,      type=UpperCAmelCase     ,      help='''help message'''		)
       expected.add_argument('''--foo_str'''     ,      nargs='''+'''     ,      default=['''Hallo''', '''Bonjour''', '''Hello''']     ,      type=UpperCAmelCase		)
       self.argparsersEqual(UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		Tuple		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   int				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   List[Any]				  ={
           '''foo''': 12,
           '''bar''': 3.1_4,
           '''baz''': '''42''',
           '''flag''': True,
       }

       lowercase					:   Tuple				  =parser.parse_dict(UpperCAmelCase		)[0]
       lowercase					:   Optional[Any]				  =BasicExample(**UpperCAmelCase		)
       self.assertEqual(UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		Optional[Any]		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   Optional[int]				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Tuple				  ={
           '''foo''': 12,
           '''bar''': 3.1_4,
           '''baz''': '''42''',
           '''flag''': True,
           '''extra''': 42,
       }

       self.assertRaises(UpperCAmelCase     ,      parser.parse_dict     ,      UpperCAmelCase     ,      allow_extra_keys=UpperCAmelCase		)






      def 				A__      ( self			:		Dict		)  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Tuple				  ={
           '''foo''': 12,
           '''bar''': 3.1_4,
           '''baz''': '''42''',
           '''flag''': True,
       }
       with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase					:   int				  =os.path.join(UpperCAmelCase     ,      '''temp_json'''		)
        os.mkdir(UpperCAmelCase		)
        with open(temp_local_path + '''.json'''     ,      '''w+'''		) as f:
         json.dump(UpperCAmelCase     ,      UpperCAmelCase		)
        lowercase					:   Union[str, Any]				  =parser.parse_yaml_file(Path(temp_local_path + '''.json'''		)		)[0]

       lowercase					:   str				  =BasicExample(**UpperCAmelCase		)
       self.assertEqual(UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		List[str]		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   str				  =HfArgumentParser(UpperCAmelCase		)

       lowercase					:   Dict				  ={
           '''foo''': 12,
           '''bar''': 3.1_4,
           '''baz''': '''42''',
           '''flag''': True,
       }
       with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase					:   Union[str, Any]				  =os.path.join(UpperCAmelCase     ,      '''temp_yaml'''		)
        os.mkdir(UpperCAmelCase		)
        with open(temp_local_path + '''.yaml'''     ,      '''w+'''		) as f:
         yaml.dump(UpperCAmelCase     ,      UpperCAmelCase		)
        lowercase					:   Union[str, Any]				  =parser.parse_yaml_file(Path(temp_local_path + '''.yaml'''		)		)[0]
       lowercase					:   str				  =BasicExample(**UpperCAmelCase		)
       self.assertEqual(UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		Tuple		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   Optional[int]				  =HfArgumentParser(UpperCAmelCase		)
       self.assertIsNotNone(UpperCAmelCase		)











'''simple docstring'''



from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


SCREAMING_SNAKE_CASE						=	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE						=	{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE						=	{
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

SCREAMING_SNAKE_CASE						=	{'allegro/herbert-base-cased': 514}
SCREAMING_SNAKE_CASE						=	{}
class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			VOCAB_FILES_NAMES
      UpperCamelCase_								=			PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_								=			PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase_								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_								=			HerbertTokenizer






      def __init__( self			:		Dict     ,      UpperCAmelCase			:		Optional[Any]=None     ,      UpperCAmelCase			:		Optional[int]=None     ,      UpperCAmelCase			:		Any=None     ,      UpperCAmelCase			:		Tuple="<s>"     ,      UpperCAmelCase			:		List[Any]="<unk>"     ,      UpperCAmelCase			:		str="<pad>"     ,      UpperCAmelCase			:		Optional[Any]="<mask>"     ,      UpperCAmelCase			:		List[str]="</s>"     ,      **UpperCAmelCase			:		List[str]     ,      )  ->  Dict:


       '''simple docstring'''

       super().__init__(
           UpperCAmelCase     ,      UpperCAmelCase     ,      tokenizer_file=UpperCAmelCase     ,      cls_token=UpperCAmelCase     ,      unk_token=UpperCAmelCase     ,      pad_token=UpperCAmelCase     ,      mask_token=UpperCAmelCase     ,      sep_token=UpperCAmelCase     ,      **UpperCAmelCase     ,      )






      def 				A__      ( self			:		int     ,      UpperCAmelCase			:		List[int]     ,      UpperCAmelCase			:		Optional[List[int]] = None		)  ->  List[int]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =[self.cls_token_id]
       lowercase					:   Any				  =[self.sep_token_id]
       if token_ids_a is None:
        return cls + token_ids_a + sep

       return cls + token_ids_a + sep + token_ids_a + sep






      def 				A__      ( self			:		Any     ,      UpperCAmelCase			:		List[int]     ,      UpperCAmelCase			:		Optional[List[int]] = None     ,      UpperCAmelCase			:		bool = False		)  ->  List[int]:


       '''simple docstring'''

       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=UpperCAmelCase     ,      token_ids_a=UpperCAmelCase     ,      already_has_special_tokens=UpperCAmelCase		)

       if token_ids_a is None:
        return [1] + ([0] * len(UpperCAmelCase		)) + [1]
       return [1] + ([0] * len(UpperCAmelCase		)) + [1] + ([0] * len(UpperCAmelCase		)) + [1]






      def 				A__      ( self			:		Optional[int]     ,      UpperCAmelCase			:		List[int]     ,      UpperCAmelCase			:		Optional[List[int]] = None		)  ->  List[int]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =[self.sep_token_id]
       lowercase					:   Dict				  =[self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
       return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






      def 				A__      ( self			:		Union[str, Any]     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		Optional[str] = None		)  ->  Tuple[str]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self._tokenizer.model.save(UpperCAmelCase     ,      name=UpperCAmelCase		)
       return tuple(UpperCAmelCase		)



