'''simple docstring'''




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
_A								=   {'LayoutLMv2Config', 'LayoutLMv3Config'}


@is_pipeline_test
class       UpperCAmelCase__  (       unittest.TestCase			):






   """simple docstring"""
   A							:       List[Any]     =    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
   A							:       Union[str, Any]     =    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

   if model_mapping is not None:
      A							:       Dict     =    {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
   if tf_model_mapping is not None:
      A							:       List[str]     =    {
          config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
      }
   @require_torch
   def 		_lowerCamelCase (self				)    ->     Any:
        lowercase_					:       Optional[Any]     				=					pipeline(
            task='text-classification'       ,				model='hf-internal-testing/tiny-random-distilbert'       ,				framework='pt'				)

        lowercase_					:       Tuple     				=					text_classifier('This is great !'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'LABEL_0', 'score': 0.5_04}]				)

        lowercase_					:       int     				=					text_classifier('This is great !'       ,				top_k=2				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}]				)

        lowercase_					:       List[str]     				=					text_classifier(['This is great !', 'This is bad']       ,				top_k=2				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[
                [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
                [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
            ]       ,				)

        lowercase_					:       int     				=					text_classifier('This is great !'       ,				top_k=1				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'LABEL_0', 'score': 0.5_04}]				)

        # Legacy behavior
        lowercase_					:       Optional[Any]     				=					text_classifier('This is great !'       ,				return_all_scores=_a				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'LABEL_0', 'score': 0.5_04}]				)

        lowercase_					:       List[str]     				=					text_classifier('This is great !'       ,				return_all_scores=_a				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[[{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}]]				)

        lowercase_					:       Union[str, Any]     				=					text_classifier(['This is great !', 'Something else']       ,				return_all_scores=_a				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[
                [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
                [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
            ]       ,				)

        lowercase_					:       List[str]     				=					text_classifier(['This is great !', 'Something else']       ,				return_all_scores=_a				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[
                {'label': 'LABEL_0', 'score': 0.5_04},
                {'label': 'LABEL_0', 'score': 0.5_04},
            ]       ,				)
   @require_torch
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        import torch

        lowercase_					:       Tuple     				=					pipeline(
            task='text-classification'       ,				model='hf-internal-testing/tiny-random-distilbert'       ,				framework='pt'       ,				device=torch.device('cpu'				)       ,				)

        lowercase_					:       str     				=					text_classifier('This is great !'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'LABEL_0', 'score': 0.5_04}]				)
   @require_tf
   def 		_lowerCamelCase (self				)    ->     Optional[Any]:
        lowercase_					:       Any     				=					pipeline(
            task='text-classification'       ,				model='hf-internal-testing/tiny-random-distilbert'       ,				framework='tf'				)

        lowercase_					:       Tuple     				=					text_classifier('This is great !'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'LABEL_0', 'score': 0.5_04}]				)
   @slow
   @require_torch
   def 		_lowerCamelCase (self				)    ->     Tuple:
        lowercase_					:       str     				=					pipeline('text-classification'				)

        lowercase_					:       str     				=					text_classifier('This is great !'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'POSITIVE', 'score': 1.0}]				)
        lowercase_					:       Tuple     				=					text_classifier('This is bad !'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'NEGATIVE', 'score': 1.0}]				)
        lowercase_					:       Optional[int]     				=					text_classifier('Birds are a type of animal'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'POSITIVE', 'score': 0.9_88}]				)
   @slow
   @require_tf
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        lowercase_					:       Optional[Any]     				=					pipeline('text-classification'       ,				framework='tf'				)

        lowercase_					:       str     				=					text_classifier('This is great !'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'POSITIVE', 'score': 1.0}]				)
        lowercase_					:       Dict     				=					text_classifier('This is bad !'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'NEGATIVE', 'score': 1.0}]				)
        lowercase_					:       str     				=					text_classifier('Birds are a type of animal'				)
        self.assertEqual(nested_simplify(_a				)       ,				[{'label': 'POSITIVE', 'score': 0.9_88}]				)
   def 		_lowerCamelCase (self       ,				_a       ,				_a       ,				_a				)    ->     Dict:
        lowercase_					:       List[str]     				=					TextClassificationPipeline(model=_a       ,				tokenizer=_a				)
        return text_classifier, ["HuggingFace is in", "This is another test"]





   def 		_lowerCamelCase (self       ,				_a       ,				_a				)    ->     Optional[int]:
        lowercase_					:       Dict     				=					text_classifier.model
        # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
        lowercase_					:       int     				=					"""HuggingFace is in"""
        lowercase_					:       Optional[Any]     				=					text_classifier(_a				)

        self.assertEqual(nested_simplify(_a				)       ,				[{'label': ANY(_a				), 'score': ANY(_a				)}]				)
        self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()				)

        lowercase_					:       List[Any]     				=					["""HuggingFace is in """, """Paris is in France"""]
        lowercase_					:       Dict     				=					text_classifier(_a				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[{'label': ANY(_a				), 'score': ANY(_a				)}, {'label': ANY(_a				), 'score': ANY(_a				)}]       ,				)
        self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()				)
        self.assertTrue(outputs[1]['label'] in model.config.idalabel.values()				)

        # Forcing to get all results with `top_k=None`
        # This is NOT the legacy format
        lowercase_					:       Any     				=					text_classifier(_a       ,				top_k=_a				)
        lowercase_					:       Dict     				=					len(model.config.idalabel.values()				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[[{'label': ANY(_a				), 'score': ANY(_a				)}] * N, [{'label': ANY(_a				), 'score': ANY(_a				)}] * N]       ,				)

        lowercase_					:       Tuple     				=					{"""text""": """HuggingFace is in """, """text_pair""": """Paris is in France"""}
        lowercase_					:       Dict     				=					text_classifier(_a				)
        self.assertEqual(
            nested_simplify(_a				)       ,				{'label': ANY(_a				), 'score': ANY(_a				)}       ,				)
        self.assertTrue(outputs['label'] in model.config.idalabel.values()				)

        # This might be used a text pair, but tokenizer + pipe interaction
        # makes it hard to understand that it's not using the pair properly
        # https://github.com/huggingface/transformers/issues/17305
        # We disabled this usage instead as it was outputting wrong outputs.
        lowercase_					:       Union[str, Any]     				=					[["""HuggingFace is in """, """Paris is in France"""]]
        with self.assertRaises(_a				):
             text_classifier(_a				)

        # This used to be valid for doing text pairs
        # We're keeping it working because of backward compatibility
        lowercase_					:       Any     				=					text_classifier([[['HuggingFace is in ', 'Paris is in France']]]				)
        self.assertEqual(
            nested_simplify(_a				)       ,				[{'label': ANY(_a				), 'score': ANY(_a				)}]       ,				)
        self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()				)





'''simple docstring'''




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin


class       UpperCAmelCase__  (       _snake_case					,     unittest.TestCase			):






   """simple docstring"""
   A							:       str     =    TransfoXLTokenizer
   A							:       Optional[int]     =    False
   A							:       List[str]     =    False
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        super().setUp()

        lowercase_					:       Any     				=					[
            '<unk>',
            '[CLS]',
            '[SEP]',
            'want',
            'unwanted',
            'wa',
            'un',
            'running',
            ',',
            'low',
            'l',
        ]
        lowercase_					:       Any     				=					os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['vocab_file']				)
        with open(self.vocab_file       ,				'w'       ,				encoding='utf-8'				) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)
   def 		_lowerCamelCase (self       ,				**_a				)    ->     Union[str, Any]:
        lowercase_					:       str     				=					True
        return TransfoXLTokenizer.from_pretrained(self.tmpdirname       ,				**_a				)
   def 		_lowerCamelCase (self       ,				_a				)    ->     Union[str, Any]:
        lowercase_					:       int     				=					'<unk> UNwanted , running'
        lowercase_					:       int     				=					'<unk> unwanted, running'
        return input_text, output_text
   def 		_lowerCamelCase (self				)    ->     Tuple:
        lowercase_					:       Any     				=					TransfoXLTokenizer(vocab_file=self.vocab_file       ,				lower_case=_a				)

        lowercase_					:       Optional[Any]     				=					tokenizer.tokenize('<unk> UNwanted , running'				)
        self.assertListEqual(_a       ,				['<unk>', 'unwanted', ',', 'running']				)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)       ,				[0, 4, 8, 7]				)
   def 		_lowerCamelCase (self				)    ->     Union[str, Any]:
        lowercase_					:       List[str]     				=					TransfoXLTokenizer(lower_case=_a				)

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '				)       ,				['hello', '!', 'how', 'are', 'you', '?']				)
   def 		_lowerCamelCase (self				)    ->     int:
        lowercase_					:       Dict     				=					TransfoXLTokenizer(lower_case=_a				)

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '				)       ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)
   def 		_lowerCamelCase (self				)    ->     Optional[Any]:
        lowercase_					:       Dict     				=					TransfoXLTokenizer(lower_case=_a				)
        lowercase_					:       int     				=					'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
        lowercase_					:       List[str]     				=					[
            'Hello',
            '(',
            'bracket',
            ')',
            'and',
            'side',
            '@-@',
            'scrolled',
            '[',
            'and',
            ']',
            'Henry',
            '\'s',
            '$',
            '5',
            '@,@',
            '000',
            'with',
            '3',
            '@.@',
            '34',
            'm',
            '.',
            'What',
            '\'s',
            'up',
            '!',
            '?',
        ]

        self.assertListEqual(tokenizer.tokenize(_a				)       ,				_a				)

        self.assertEqual(tokenizer.convert_tokens_to_string(_a				)       ,				_a				)





   def 		_lowerCamelCase (self				)    ->     Any:
        lowercase_					:       Optional[int]     				=					self.get_tokenizer()
        lowercase_					:       str     				=					len(_a				)

        tokenizer.add_tokens(['new1', 'new2']				)
        tokenizer.move_added_token('new1'       ,				1				)

        # Check that moved token is not copied (duplicate)
        self.assertEqual(len(_a				)       ,				original_len + 2				)
        # Check that token is moved to specified id
        self.assertEqual(tokenizer.encode('new1'				)       ,				[1]				)
        self.assertEqual(tokenizer.decode([1]				)       ,				'new1'				)




