




import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin



@require_jieba
class     _A   (    __UpperCamelCase	, unittest.TestCase							):
  SCREAMING_SNAKE_CASE_	:	Union[str, Any]						=CpmAntTokenizer
  SCREAMING_SNAKE_CASE_	:	Tuple						=False






  def 		_a					(self	)		->      Tuple:





       '''simple docstring'''





       super().setUp()

       UpperCamelCase__										=      [
           '''<d>''',
           '''</d>''',
           '''<s>''',
           '''</s>''',
           '''</_>''',
           '''<unk>''',
           '''<pad>''',
           '''</n>''',
           '''我''',
           '''是''',
           '''C''',
           '''P''',
           '''M''',
           '''A''',
           '''n''',
           '''t''',
       ]
       UpperCamelCase__										=      os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file''']	)
       with open(self.vocab_file   ,				'''w'''   ,				encoding='''utf-8'''	) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)






  @tooslow
  def 		_a					(self	)		->      Tuple:





       '''simple docstring'''





       UpperCamelCase__										=      CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''	)
       UpperCamelCase__										=      '''今天天气真好！'''
       UpperCamelCase__										=      ['''今天''', '''天气''', '''真''', '''好''', '''！''']
       UpperCamelCase__										=      tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
       self.assertListEqual(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      '''今天天气真好！'''
       UpperCamelCase__										=      [tokenizer.bos_token] + tokens

       UpperCamelCase__										=      [6, 9802, 1_4962, 2082, 831, 244]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)   ,				SCREAMING_SNAKE_CASE_	)

       UpperCamelCase__										=      tokenizer.decode(SCREAMING_SNAKE_CASE_	)
       self.assertEqual(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
     import torch

     from transformers.activations import gelu_new, gelu_python, get_activation



@require_torch
class     _A   (    unittest.TestCase							):






  def 		_a					(self	)		->      str:





       '''simple docstring'''





       UpperCamelCase__										=      torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]	)
       UpperCamelCase__										=      get_activation('''gelu'''	)
       self.assertTrue(torch.allclose(gelu_python(SCREAMING_SNAKE_CASE_	)   ,				torch_builtin(SCREAMING_SNAKE_CASE_	)	)	)
       self.assertFalse(torch.allclose(gelu_python(SCREAMING_SNAKE_CASE_	)   ,				gelu_new(SCREAMING_SNAKE_CASE_	)	)	)






  def 		_a					(self	)		->      str:





       '''simple docstring'''





       UpperCamelCase__										=      torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]	)
       UpperCamelCase__										=      get_activation('''gelu'''	)
       UpperCamelCase__										=      get_activation('''gelu_10'''	)

       UpperCamelCase__										=      torch_builtin(SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      geluaa(SCREAMING_SNAKE_CASE_	)

       UpperCamelCase__										=      torch.where(y_gelu_aa < 10.0   ,				1   ,				0	)

       self.assertTrue(torch.max(SCREAMING_SNAKE_CASE_	).item() == 10.0	)
       self.assertTrue(torch.allclose(y_gelu * clipped_mask   ,				y_gelu_aa * clipped_mask	)	)






  def 		_a					(self	)		->      Optional[Any]:





       '''simple docstring'''





       get_activation('''gelu'''	)
       get_activation('''gelu_10'''	)
       get_activation('''gelu_fast'''	)
       get_activation('''gelu_new'''	)
       get_activation('''gelu_python'''	)
       get_activation('''gelu_pytorch_tanh'''	)
       get_activation('''linear'''	)
       get_activation('''mish'''	)
       get_activation('''quick_gelu'''	)
       get_activation('''relu'''	)
       get_activation('''sigmoid'''	)
       get_activation('''silu'''	)
       get_activation('''swish'''	)
       get_activation('''tanh'''	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            get_activation('''bogus'''	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            get_activation(SCREAMING_SNAKE_CASE_	)






  def 		_a					(self	)		->      Optional[Any]:





       '''simple docstring'''





       UpperCamelCase__										=      get_activation('''gelu'''	)
       UpperCamelCase__										=      1
       UpperCamelCase__										=      get_activation('''gelu'''	)
       self.assertEqual(acta.a   ,				1	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            UpperCamelCase__										=      acta.a



