
import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase      =get_tests_dir("fixtures/test_sentencepiece.model")




@require_sentencepiece
@require_tokenizers
class 				_lowerCamelCase (      UpperCamelCase_	,							unittest.TestCase ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  XLNetTokenizer
     SCREAMING_SNAKE_CASE_		   =  XLNetTokenizerFast
     SCREAMING_SNAKE_CASE_		   =  True
     SCREAMING_SNAKE_CASE_		   =  True






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Any:




           """simple docstring"""







           super().setUp()

           # We have a SentencePiece fixture for testing
           UpperCamelCase__    :   Optional[Any]					=				XLNetTokenizer(__SCREAMING_SNAKE_CASE						,						keep_accents=__SCREAMING_SNAKE_CASE  )
           tokenizer.sanitize_special_tokens()
           tokenizer.save_pretrained(self.tmpdirname  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Dict:




           """simple docstring"""







           UpperCamelCase__    :   str					=				'''<s>'''
           UpperCamelCase__    :   List[str]					=				1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE  )						,						__SCREAMING_SNAKE_CASE  )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE  )						,						__SCREAMING_SNAKE_CASE  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   int:




           """simple docstring"""







           UpperCamelCase__    :   Optional[int]					=				list(self.get_tokenizer().get_vocab().keys()  )

           self.assertEqual(vocab_keys[0]						,						'''<unk>'''  )
           self.assertEqual(vocab_keys[1]						,						'''<s>'''  )
           self.assertEqual(vocab_keys[-1]						,						'''<eod>'''  )
           self.assertEqual(len(__SCREAMING_SNAKE_CASE  )						,						1_0_0_6  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   int:




           """simple docstring"""







           self.assertEqual(self.get_tokenizer().vocab_size						,						1_0_0_0  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   str					=				XLNetTokenizer(__SCREAMING_SNAKE_CASE						,						keep_accents=__SCREAMING_SNAKE_CASE  )

           UpperCamelCase__    :   Tuple					=				tokenizer.tokenize('''This is a test'''  )
           self.assertListEqual(__SCREAMING_SNAKE_CASE						,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']  )

           self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE  )						,						[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]  )

           UpperCamelCase__    :   Dict					=				tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE						,						[
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''9''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''é''',
                   '''.''',
               ]						,						)
           UpperCamelCase__    :   Any					=				tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE  )
           self.assertListEqual(__SCREAMING_SNAKE_CASE						,						[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]  )

           UpperCamelCase__    :   Union[str, Any]					=				tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE  )
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE						,						[
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''<unk>''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''<unk>''',
                   '''.''',
               ]						,						)






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   str:




           """simple docstring"""







           UpperCamelCase__    :   int					=				XLNetTokenizer(__SCREAMING_SNAKE_CASE						,						do_lower_case=__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Optional[Any]					=				tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE						,						[
                   SPIECE_UNDERLINE + '''''',
                   '''i''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''9''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''se''',
                   '''.''',
               ]						,						)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )						,						['''▁he''', '''ll''', '''o''']  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   str					=				XLNetTokenizer(__SCREAMING_SNAKE_CASE						,						do_lower_case=__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Optional[int]					=				tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE						,						[
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''9''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''se''',
                   '''.''',
               ]						,						)






     @slow
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[str]:




           """simple docstring"""







           UpperCamelCase__    :   Dict					=				XLNetTokenizer.from_pretrained('''xlnet-base-cased'''  )

           UpperCamelCase__    :   Union[str, Any]					=				tokenizer.encode('''sequence builders'''						,						add_special_tokens=__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Optional[int]					=				tokenizer.encode('''multi-sequence build'''						,						add_special_tokens=__SCREAMING_SNAKE_CASE  )

           UpperCamelCase__    :   Any					=				tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Tuple					=				tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )

           assert encoded_sentence == text + [4, 3]
           assert encoded_pair == text + [4] + text_a + [4, 3]






     @slow
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           UpperCamelCase__    :   List[Any]					=				{'''input_ids''': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=__SCREAMING_SNAKE_CASE						,						model_name='''xlnet-base-cased'''						,						revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''						,						)







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase      ={
    "configuration_clipseg": [
        "CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "CLIPSegConfig",
        "CLIPSegTextConfig",
        "CLIPSegVisionConfig",
    ],
    "processing_clipseg": ["CLIPSegProcessor"],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase      =[
        "CLIPSEG_PRETRAINED_MODEL_ARCHIVE_LIST",
        "CLIPSegModel",
        "CLIPSegPreTrainedModel",
        "CLIPSegTextModel",
        "CLIPSegVisionModel",
        "CLIPSegForImageSegmentation",
    ]

if TYPE_CHECKING:
    from .configuration_clipseg import (
        CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP,
        CLIPSegConfig,
        CLIPSegTextConfig,
        CLIPSegVisionConfig,
    )
    from .processing_clipseg import CLIPSegProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_clipseg import (
            CLIPSEG_PRETRAINED_MODEL_ARCHIVE_LIST,
            CLIPSegForImageSegmentation,
            CLIPSegModel,
            CLIPSegPreTrainedModel,
            CLIPSegTextModel,
            CLIPSegVisionModel,
        )

else:
    import sys

    lowerCamelCase      =_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


