






"""simple docstring"""





import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin


@require_jieba
class      _lowercase (     lowerCAmelCase		,  unittest.TestCase						):
      _a   :     Dict								=   CpmAntTokenizer
      _a   :     Dict								=   False


      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            super().setUp()

            __snake_case      :    int  			=[
                '''<d>''',
                '''</d>''',
                '''<s>''',
                '''</s>''',
                '''</_>''',
                '''<unk>''',
                '''<pad>''',
                '''</n>''',
                '''我''',
                '''是''',
                '''C''',
                '''P''',
                '''M''',
                '''A''',
                '''n''',
                '''t''',
            ]
            __snake_case      :    List[Any]  			=os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['''vocab_file''']						)
            with open(self.vocab_file							,   '''w'''							,   encoding='''utf-8'''						) as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)







      @tooslow
      def _UpperCamelCase						(    self				:	Optional[int]						):





            """simple docstring"""







            __snake_case      :    int  			=CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''						)
            __snake_case      :    Any  			='''今天天气真好！'''
            __snake_case      :    Any  			=['''今天''', '''天气''', '''真''', '''好''', '''！''']
            __snake_case      :    Optional[Any]  			=tokenizer.tokenize(a						)
            self.assertListEqual(a							,   a						)
            __snake_case      :    List[Any]  			='''今天天气真好！'''
            __snake_case      :    int  			=[tokenizer.bos_token] + tokens

            __snake_case      :    Any  			=[6, 9_8_0_2, 1_4_9_6_2, 2_0_8_2, 8_3_1, 2_4_4]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a						)							,   a						)

            __snake_case      :    Tuple  			=tokenizer.decode(a						)
            self.assertEqual(a							,   a						)










"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_	:				Optional[int]										= logging.get_logger(__name__)

UpperCamelCase_	:				Union[str, Any]										= {"""ctrl""": """https://huggingface.co/ctrl/resolve/main/config.json"""}


class      _lowercase (     lowerCAmelCase						):
      _a   :     str								=   '''ctrl'''
      _a   :     Optional[int]								=   ['''past_key_values''']
      _a   :     List[Any]								=   {
          '''max_position_embeddings''': '''n_positions''',
          '''hidden_size''': '''n_embd''',
          '''num_attention_heads''': '''n_head''',
          '''num_hidden_layers''': '''n_layer''',
      }


      def __init__(    self				:	str							,   a				:	Optional[int]=2_4_6_5_3_4							,   a				:	Optional[int]=2_5_6							,   a				:	Optional[int]=1_2_8_0							,   a				:	Any=8_1_9_2							,   a				:	int=4_8							,   a				:	Optional[Any]=1_6							,   a				:	str=0.1							,   a				:	str=0.1							,   a				:	Any=1e-6							,   a				:	Optional[int]=0.0_2							,   a				:	int=True							,   **a				:	str							,   ):





            """simple docstring"""







            __snake_case      :    Tuple  			=vocab_size
            __snake_case      :    Optional[Any]  			=n_positions
            __snake_case      :    List[Any]  			=n_embd
            __snake_case      :    Any  			=n_layer
            __snake_case      :    Any  			=n_head
            __snake_case      :    str  			=dff
            __snake_case      :    List[str]  			=resid_pdrop
            __snake_case      :    str  			=embd_pdrop
            __snake_case      :    Union[str, Any]  			=layer_norm_epsilon
            __snake_case      :    List[Any]  			=initializer_range

            __snake_case      :    Any  			=use_cache

            super().__init__(**a						)


