




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class     _A   (    __UpperCamelCase	, unittest.TestCase							):
  SCREAMING_SNAKE_CASE_	:	Dict						=CLIPTokenizer
  SCREAMING_SNAKE_CASE_	:	int						=CLIPTokenizerFast
  SCREAMING_SNAKE_CASE_	:	Optional[int]						=True
  SCREAMING_SNAKE_CASE_	:	int						={}
  SCREAMING_SNAKE_CASE_	:	Optional[int]						=False






  def 		_a					(self	)		->      Optional[int]:





       '''simple docstring'''





       super().setUp()

       # fmt: off
       UpperCamelCase__										=      ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
       # fmt: on
       UpperCamelCase__										=      dict(zip(SCREAMING_SNAKE_CASE_   ,				range(len(SCREAMING_SNAKE_CASE_	)	)	)	)
       UpperCamelCase__										=      ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
       UpperCamelCase__										=      {'''unk_token''': '''<unk>'''}

       UpperCamelCase__										=      os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file''']	)
       UpperCamelCase__										=      os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file''']	)
       with open(self.vocab_file   ,				'''w'''   ,				encoding='''utf-8'''	) as fp:
            fp.write(json.dumps(SCREAMING_SNAKE_CASE_	) + '''\n'''	)
       with open(self.merges_file   ,				'''w'''   ,				encoding='''utf-8'''	) as fp:
            fp.write('''\n'''.join(SCREAMING_SNAKE_CASE_	)	)






  def 		_a					(self   ,				**SCREAMING_SNAKE_CASE_	)		->      Optional[int]:





       '''simple docstring'''





       kwargs.update(self.special_tokens_map	)
       return CLIPTokenizer.from_pretrained(self.tmpdirname   ,				**SCREAMING_SNAKE_CASE_	)






  def 		_a					(self   ,				**SCREAMING_SNAKE_CASE_	)		->      str:





       '''simple docstring'''





       kwargs.update(self.special_tokens_map	)
       return CLIPTokenizerFast.from_pretrained(self.tmpdirname   ,				**SCREAMING_SNAKE_CASE_	)






  def 		_a					(self   ,				SCREAMING_SNAKE_CASE_	)		->      List[str]:





       '''simple docstring'''





       UpperCamelCase__										=      '''lower newer'''
       UpperCamelCase__										=      '''lower newer'''
       return input_text, output_text






  def 		_a					(self	)		->      int:





       '''simple docstring'''





       UpperCamelCase__										=      CLIPTokenizer(self.vocab_file   ,				self.merges_file   ,				**self.special_tokens_map	)
       UpperCamelCase__										=      '''lower newer'''
       UpperCamelCase__										=      ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
       UpperCamelCase__										=      tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
       self.assertListEqual(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)

       UpperCamelCase__										=      tokens + [tokenizer.unk_token]
       UpperCamelCase__										=      [10, 2, 16, 9, 3, 2, 16, 20]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)   ,				SCREAMING_SNAKE_CASE_	)






  @require_ftfy
  def 		_a					(self	)		->      Tuple:





       '''simple docstring'''





       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
                 UpperCamelCase__										=      self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_   ,				**SCREAMING_SNAKE_CASE_	)
                 UpperCamelCase__										=      self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_   ,				**SCREAMING_SNAKE_CASE_	)

                 UpperCamelCase__										=      '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                 UpperCamelCase__										=      tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_	)
                 UpperCamelCase__										=      tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_	)

                 self.assertListEqual(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)

                 # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                 # with Tilde) encoded in 2 different ways
                 UpperCamelCase__										=      '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                 UpperCamelCase__										=      tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_	)
                 UpperCamelCase__										=      tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_	)

                 self.assertListEqual(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)

                 # Test that the tokenization is identical on unicode of space type
                 UpperCamelCase__										=      [
                     '''\u0009''',  # (horizontal tab, '\t')
                     '''\u000B''',  # (vertical tab)
                     '''\u000C''',  # (form feed)
                     '''\u0020''',  # (space, ' ')
                     '''\u200E''',  # (left-to-right mark):w
                     '''\u200F''',  # (right-to-left mark)
                 ]
                 for unicode_seq in spaces_unicodes:
                      UpperCamelCase__										=      tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_	)
                      UpperCamelCase__										=      tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_	)

                      self.assertListEqual(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)

                 # Test that the tokenization is identical on unicode of line break type
                 UpperCamelCase__										=      [
                     '''\u000A''',  # (line feed, '\n')
                     '''\r\n''',  # (carriage return and line feed, '\r\n')
                     '''\u000D''',  # (carriage return, '\r')
                     '''\r''',  # (carriage return, '\r')
                     '''\u000D''',  # (carriage return, '\r')
                     '''\u2028''',  # (line separator)
                     '''\u2029''',  # (paragraph separator)
                     # "\u0085", # (next line)
                 ]

                 # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                 # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                 # space (and thus into an empty list).

                 for unicode_seq in line_break_unicodes:
                      UpperCamelCase__										=      tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_	)
                      UpperCamelCase__										=      tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_	)

                      self.assertListEqual(SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_	)






  def 		_a					(self	)		->      Any:





       '''simple docstring'''





       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
                 UpperCamelCase__										=      '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                 UpperCamelCase__										=      F"{text_of_1_token} {text_of_1_token}"

                 UpperCamelCase__										=      self.rust_tokenizer_class.from_pretrained(
                     SCREAMING_SNAKE_CASE_   ,				use_fast=SCREAMING_SNAKE_CASE_   ,				)
                 UpperCamelCase__										=      tokenizer_r(SCREAMING_SNAKE_CASE_   ,				return_offsets_mapping=SCREAMING_SNAKE_CASE_   ,				add_special_tokens=SCREAMING_SNAKE_CASE_	)
                 self.assertEqual(encoding.offset_mapping[0]   ,				(0, len(SCREAMING_SNAKE_CASE_	))	)
                 self.assertEqual(
                     encoding.offset_mapping[1]   ,				(len(SCREAMING_SNAKE_CASE_	) + 1, len(SCREAMING_SNAKE_CASE_	) + 1 + len(SCREAMING_SNAKE_CASE_	))   ,				)

                 UpperCamelCase__										=      F" {text}"

                 UpperCamelCase__										=      self.rust_tokenizer_class.from_pretrained(
                     SCREAMING_SNAKE_CASE_   ,				use_fast=SCREAMING_SNAKE_CASE_   ,				)
                 UpperCamelCase__										=      tokenizer_r(SCREAMING_SNAKE_CASE_   ,				return_offsets_mapping=SCREAMING_SNAKE_CASE_   ,				add_special_tokens=SCREAMING_SNAKE_CASE_	)
                 self.assertEqual(encoding.offset_mapping[0]   ,				(1, 1 + len(SCREAMING_SNAKE_CASE_	))	)
                 self.assertEqual(
                     encoding.offset_mapping[1]   ,				(1 + len(SCREAMING_SNAKE_CASE_	) + 1, 1 + len(SCREAMING_SNAKE_CASE_	) + 1 + len(SCREAMING_SNAKE_CASE_	))   ,				)






  def 		_a					(self	)		->      List[Any]:





       '''simple docstring'''





       with self.assertRaises(SCREAMING_SNAKE_CASE_	) as context:
            self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''	)

       self.assertTrue(
           context.exception.args[0].startswith(
               '''The `backend_tokenizer` provided does not match the expected format.'''	)	)






  @require_ftfy
  def 		_a					(self	)		->      Union[str, Any]:





       '''simple docstring'''





       super().test_tokenization_python_rust_equals()






  def 		_a					(self	)		->      Any:





       '''simple docstring'''





       pass




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
     import torch

     from transformers.activations import gelu_new, gelu_python, get_activation



@require_torch
class     _A   (    unittest.TestCase							):






  def 		_a					(self	)		->      str:





       '''simple docstring'''





       UpperCamelCase__										=      torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]	)
       UpperCamelCase__										=      get_activation('''gelu'''	)
       self.assertTrue(torch.allclose(gelu_python(SCREAMING_SNAKE_CASE_	)   ,				torch_builtin(SCREAMING_SNAKE_CASE_	)	)	)
       self.assertFalse(torch.allclose(gelu_python(SCREAMING_SNAKE_CASE_	)   ,				gelu_new(SCREAMING_SNAKE_CASE_	)	)	)






  def 		_a					(self	)		->      str:





       '''simple docstring'''





       UpperCamelCase__										=      torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]	)
       UpperCamelCase__										=      get_activation('''gelu'''	)
       UpperCamelCase__										=      get_activation('''gelu_10'''	)

       UpperCamelCase__										=      torch_builtin(SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      geluaa(SCREAMING_SNAKE_CASE_	)

       UpperCamelCase__										=      torch.where(y_gelu_aa < 10.0   ,				1   ,				0	)

       self.assertTrue(torch.max(SCREAMING_SNAKE_CASE_	).item() == 10.0	)
       self.assertTrue(torch.allclose(y_gelu * clipped_mask   ,				y_gelu_aa * clipped_mask	)	)






  def 		_a					(self	)		->      Optional[Any]:





       '''simple docstring'''





       get_activation('''gelu'''	)
       get_activation('''gelu_10'''	)
       get_activation('''gelu_fast'''	)
       get_activation('''gelu_new'''	)
       get_activation('''gelu_python'''	)
       get_activation('''gelu_pytorch_tanh'''	)
       get_activation('''linear'''	)
       get_activation('''mish'''	)
       get_activation('''quick_gelu'''	)
       get_activation('''relu'''	)
       get_activation('''sigmoid'''	)
       get_activation('''silu'''	)
       get_activation('''swish'''	)
       get_activation('''tanh'''	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            get_activation('''bogus'''	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            get_activation(SCREAMING_SNAKE_CASE_	)






  def 		_a					(self	)		->      Optional[Any]:





       '''simple docstring'''





       UpperCamelCase__										=      get_activation('''gelu'''	)
       UpperCamelCase__										=      1
       UpperCamelCase__										=      get_activation('''gelu'''	)
       self.assertEqual(acta.a   ,				1	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            UpperCamelCase__										=      acta.a



