'''simple docstring'''

import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY
@is_pipeline_test
class   SCREAMING_SNAKE_CASE			(						unittest.TestCase):







		"""simple docstring"""
		lowercase   :      Tuple       =							MODEL_FOR_MASKED_LM_MAPPING
		lowercase   :      Any       =							TF_MODEL_FOR_MASKED_LM_MAPPING






		def 		UpperCamelCase__       (   self				) ->     int:
								super().tearDown()
								# clean-up as much as possible GPU memory occupied by PyTorch
								gc.collect()
								if is_torch_available():
														import torch

														torch.cuda.empty_cache()






		@require_tf
		def 		UpperCamelCase__       (   self				) ->     List[str]:
								_lowerCAmelCase										=pipeline(task='fill-mask'		,						model='sshleifer/tiny-distilroberta-base'		,						top_k=2		,						framework='tf'				)
								_lowerCAmelCase										=unmasker('My name is <mask>'				)
								self.assertEqual(
								    nested_simplify(__A		,						decimals=6				)		,						[
								        {'sequence': 'My name is grouped', 'score': 2.1E-05, 'token': 3_8015, 'token_str': ' grouped'},
								        {'sequence': 'My name is accuser', 'score': 2.1E-05, 'token': 2_5506, 'token_str': ' accuser'},
								    ]		,						)

								_lowerCAmelCase										=unmasker('The largest city in France is <mask>'				)
								self.assertEqual(
								    nested_simplify(__A		,						decimals=6				)		,						[
								        {
								            'sequence': 'The largest city in France is grouped',
								            'score': 2.1E-05,
								            'token': 3_8015,
								            'token_str': ' grouped',
								        },
								        {
								            'sequence': 'The largest city in France is accuser',
								            'score': 2.1E-05,
								            'token': 2_5506,
								            'token_str': ' accuser',
								        },
								    ]		,						)

								_lowerCAmelCase										=unmasker('My name is <mask>'		,						targets=[' Patrick', ' Clara', ' Teven']		,						top_k=3				)
								self.assertEqual(
								    nested_simplify(__A		,						decimals=6				)		,						[
								        {'sequence': 'My name is Clara', 'score': 2E-05, 'token': 1_3606, 'token_str': ' Clara'},
								        {'sequence': 'My name is Patrick', 'score': 2E-05, 'token': 3499, 'token_str': ' Patrick'},
								        {'sequence': 'My name is Te', 'score': 1.9E-05, 'token': 2941, 'token_str': ' Te'},
								    ]		,						)






		@require_torch
		def 		UpperCamelCase__       (   self				) ->     Tuple:
								_lowerCAmelCase										=pipeline(task='fill-mask'		,						model='sshleifer/tiny-distilroberta-base'		,						top_k=2		,						framework='pt'				)

								_lowerCAmelCase										=unmasker('My name is <mask>'				)
								self.assertEqual(
								    nested_simplify(__A		,						decimals=6				)		,						[
								        {'sequence': 'My name is Maul', 'score': 2.2E-05, 'token': 3_5676, 'token_str': ' Maul'},
								        {'sequence': 'My name isELS', 'score': 2.2E-05, 'token': 1_6416, 'token_str': 'ELS'},
								    ]		,						)

								_lowerCAmelCase										=unmasker('The largest city in France is <mask>'				)
								self.assertEqual(
								    nested_simplify(__A		,						decimals=6				)		,						[
								        {
								            'sequence': 'The largest city in France is Maul',
								            'score': 2.2E-05,
								            'token': 3_5676,
								            'token_str': ' Maul',
								        },
								        {'sequence': 'The largest city in France isELS', 'score': 2.2E-05, 'token': 1_6416, 'token_str': 'ELS'},
								    ]		,						)

								_lowerCAmelCase										=unmasker('My name is <mask>'		,						targets=[' Patrick', ' Clara', ' Teven']		,						top_k=3				)
								self.assertEqual(
								    nested_simplify(__A		,						decimals=6				)		,						[
								        {'sequence': 'My name is Patrick', 'score': 2.1E-05, 'token': 3499, 'token_str': ' Patrick'},
								        {'sequence': 'My name is Te', 'score': 2E-05, 'token': 2941, 'token_str': ' Te'},
								        {'sequence': 'My name is Clara', 'score': 2E-05, 'token': 1_3606, 'token_str': ' Clara'},
								    ]		,						)

								_lowerCAmelCase										=unmasker('My name is <mask> <mask>'		,						top_k=2				)

								self.assertEqual(
								    nested_simplify(__A		,						decimals=6				)		,						[
								        [
								            {
								                'score': 2.2E-05,
								                'token': 3_5676,
								                'token_str': ' Maul',
								                'sequence': '<s>My name is Maul<mask></s>',
								            },
								            {'score': 2.2E-05, 'token': 1_6416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
								        ],
								        [
								            {
								                'score': 2.2E-05,
								                'token': 3_5676,
								                'token_str': ' Maul',
								                'sequence': '<s>My name is<mask> Maul</s>',
								            },
								            {'score': 2.2E-05, 'token': 1_6416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
								        ],
								    ]		,						)






		@require_torch_gpu
		def 		UpperCamelCase__       (   self				) ->     Dict:
								_lowerCAmelCase										=pipeline('fill-mask'		,						model='hf-internal-testing/tiny-random-distilbert'		,						device=0		,						framework='pt'				)

								# convert model to fp16
								pipe.model.half()

								_lowerCAmelCase										=pipe('Paris is the [MASK] of France.'				)
								# We actually don't care about the result, we just want to make sure
								# it works, meaning the float16 tensor got casted back to float32
								# for postprocessing.
								self.assertIsInstance(__A		,						__A				)






		@slow
		@require_torch
		def 		UpperCamelCase__       (   self				) ->     Any:
								_lowerCAmelCase										=pipeline(task='fill-mask'		,						model='distilroberta-base'		,						top_k=2		,						framework='pt'				)
								self.run_large_test(__A				)






		@slow
		@require_tf
		def 		UpperCamelCase__       (   self				) ->     Tuple:
								_lowerCAmelCase										=pipeline(task='fill-mask'		,						model='distilroberta-base'		,						top_k=2		,						framework='tf'				)
								self.run_large_test(__A				)






		def 		UpperCamelCase__       (   self		,						__A				) ->     Any:
								_lowerCAmelCase										=unmasker('My name is <mask>'				)
								self.assertEqual(
								    nested_simplify(__A				)		,						[
								        {'sequence': 'My name is John', 'score': 0.008, 'token': 610, 'token_str': ' John'},
								        {'sequence': 'My name is Chris', 'score': 0.007, 'token': 1573, 'token_str': ' Chris'},
								    ]		,						)
								_lowerCAmelCase										=unmasker('The largest city in France is <mask>'				)
								self.assertEqual(
								    nested_simplify(__A				)		,						[
								        {
								            'sequence': 'The largest city in France is Paris',
								            'score': 0.251,
								            'token': 2201,
								            'token_str': ' Paris',
								        },
								        {
								            'sequence': 'The largest city in France is Lyon',
								            'score': 0.214,
								            'token': 1_2790,
								            'token_str': ' Lyon',
								        },
								    ]		,						)

								_lowerCAmelCase										=unmasker('My name is <mask>'		,						targets=[' Patrick', ' Clara', ' Teven']		,						top_k=3				)
								self.assertEqual(
								    nested_simplify(__A				)		,						[
								        {'sequence': 'My name is Patrick', 'score': 0.005, 'token': 3499, 'token_str': ' Patrick'},
								        {'sequence': 'My name is Clara', 'score': 0.000, 'token': 1_3606, 'token_str': ' Clara'},
								        {'sequence': 'My name is Te', 'score': 0.000, 'token': 2941, 'token_str': ' Te'},
								    ]		,						)






		@require_torch
		def 		UpperCamelCase__       (   self				) ->     Optional[int]:
								_lowerCAmelCase										=pipeline(task='fill-mask'		,						model='sshleifer/tiny-distilroberta-base'		,						framework='pt'				)
								_lowerCAmelCase										=None
								_lowerCAmelCase										=None
								self.run_pipeline_test(__A		,						[]				)






		@require_tf
		def 		UpperCamelCase__       (   self				) ->     int:
								_lowerCAmelCase										=pipeline(task='fill-mask'		,						model='sshleifer/tiny-distilroberta-base'		,						framework='tf'				)
								_lowerCAmelCase										=None
								_lowerCAmelCase										=None
								self.run_pipeline_test(__A		,						[]				)






		def 		UpperCamelCase__       (   self		,						__A		,						__A		,						__A				) ->     Tuple:
								if tokenizer is None or tokenizer.mask_token_id is None:
														self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'				)

								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A				)
								_lowerCAmelCase										=[
								    F'''This is another {tokenizer.mask_token} test''',
								]
								return fill_masker, examples






		def 		UpperCamelCase__       (   self		,						__A		,						__A				) ->     Tuple:
								_lowerCAmelCase										=fill_masker.tokenizer
								_lowerCAmelCase										=fill_masker.model

								_lowerCAmelCase										=fill_masker(
								    F'''This is a {tokenizer.mask_token}'''		,						)
								self.assertEqual(
								    __A		,						[
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								    ]		,						)

								_lowerCAmelCase										=fill_masker([F'''This is a {tokenizer.mask_token}''']				)
								self.assertEqual(
								    __A		,						[
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								    ]		,						)

								_lowerCAmelCase										=fill_masker([F'''This is a {tokenizer.mask_token}''', F'''Another {tokenizer.mask_token} great test.''']				)
								self.assertEqual(
								    __A		,						[
								        [
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        ],
								        [
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        ],
								    ]		,						)

								with self.assertRaises(__A				):
														fill_masker([None]				)
								# No mask_token is not supported
								with self.assertRaises(__A				):
														fill_masker('This is'				)

								self.run_test_top_k(__A		,						__A				)
								self.run_test_targets(__A		,						__A				)
								self.run_test_top_k_targets(__A		,						__A				)
								self.fill_mask_with_duplicate_targets_and_top_k(__A		,						__A				)
								self.fill_mask_with_multiple_masks(__A		,						__A				)






		def 		UpperCamelCase__       (   self		,						__A		,						__A				) ->     str:
								_lowerCAmelCase										=tokenizer.get_vocab()
								_lowerCAmelCase										=sorted(vocab.keys()				)[:2]
								# Pipeline argument
								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A		,						targets=__A				)
								_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''				)
								self.assertEqual(
								    __A		,						[
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								    ]		,						)
								_lowerCAmelCase										={vocab[el] for el in targets}
								self.assertEqual({el['token'] for el in outputs}		,						__A				)
								_lowerCAmelCase										=[tokenizer.decode([x]				) for x in target_ids]
								self.assertEqual({el['token_str'] for el in outputs}		,						set(__A				)				)

								# Call argument
								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A				)
								_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						targets=__A				)
								self.assertEqual(
								    __A		,						[
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								    ]		,						)
								_lowerCAmelCase										={vocab[el] for el in targets}
								self.assertEqual({el['token'] for el in outputs}		,						__A				)
								_lowerCAmelCase										=[tokenizer.decode([x]				) for x in target_ids]
								self.assertEqual({el['token_str'] for el in outputs}		,						set(__A				)				)

								# Score equivalence
								_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						targets=__A				)
								_lowerCAmelCase										=[top_mask['token_str'] for top_mask in outputs]
								_lowerCAmelCase										=[top_mask['score'] for top_mask in outputs]

								# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
								if set(__A				) == set(__A				):
														_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						targets=__A				)
														_lowerCAmelCase										=[top_mask['score'] for top_mask in unmasked_targets]
														self.assertEqual(nested_simplify(__A				)		,						nested_simplify(__A				)				)

								# Raises with invalid
								with self.assertRaises(__A				):
														_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						targets=[]				)
								# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
								if "" not in tokenizer.get_vocab():
														with self.assertRaises(__A				):
																				_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						targets=['']				)
														with self.assertRaises(__A				):
																				_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						targets=''				)






		def 		UpperCamelCase__       (   self		,						__A		,						__A				) ->     str:
								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A		,						top_k=2				)
								_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''				)
								self.assertEqual(
								    __A		,						[
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								    ]		,						)

								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A				)
								_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						top_k=2				)
								self.assertEqual(
								    __A		,						[
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								    ]		,						)
								self.assertEqual(nested_simplify(__A				)		,						nested_simplify(__A				)				)






		def 		UpperCamelCase__       (   self		,						__A		,						__A				) ->     Tuple:
								_lowerCAmelCase										=tokenizer.get_vocab()
								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A				)

								# top_k=2, ntargets=3
								_lowerCAmelCase										=sorted(vocab.keys()				)[:3]
								_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						top_k=2		,						targets=__A				)

								# If we use the most probably targets, and filter differently, we should still
								# have the same results
								_lowerCAmelCase										=[el['token_str'] for el in sorted(__A		,						key=lambda __A				: x["score"]		,						reverse=__A				)]
								# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
								if set(__A				).issubset(__A				):
														_lowerCAmelCase										=fill_masker(F'''This is a {tokenizer.mask_token}'''		,						top_k=3		,						targets=__A				)
														# They should yield exactly the same result
														self.assertEqual(nested_simplify(__A				)		,						nested_simplify(__A				)				)






		def 		UpperCamelCase__       (   self		,						__A		,						__A				) ->     Optional[Any]:
								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A				)
								_lowerCAmelCase										=tokenizer.get_vocab()
								# String duplicates + id duplicates
								_lowerCAmelCase										=sorted(vocab.keys()				)[:3]
								_lowerCAmelCase										=[targets[0], targets[1], targets[0], targets[2], targets[1]]
								_lowerCAmelCase										=fill_masker(F'''My name is {tokenizer.mask_token}'''		,						targets=__A		,						top_k=10				)

								# The target list contains duplicates, so we can't output more
								# than them
								self.assertEqual(len(__A				)		,						3				)






		def 		UpperCamelCase__       (   self		,						__A		,						__A				) ->     Union[str, Any]:
								_lowerCAmelCase										=FillMaskPipeline(model=__A		,						tokenizer=__A				)

								_lowerCAmelCase										=fill_masker(
								    F'''This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'''		,						top_k=2				)
								self.assertEqual(
								    __A		,						[
								        [
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        ],
								        [
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        ],
								        [
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								            {'sequence': ANY(__A				), 'score': ANY(__A				), 'token': ANY(__A				), 'token_str': ANY(__A				)},
								        ],
								    ]		,						)






'''simple docstring'''

import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
class   SCREAMING_SNAKE_CASE			(						__lowercase):







		"""simple docstring"""
		lowercase   :      Any       =							['image_processor', 'tokenizer']
		lowercase   :      Any       =							'CLIPImageProcessor'
		lowercase   :      int       =							('CLIPTokenizer', 'CLIPTokenizerFast')






		def __init__(   self		,						__A=None		,						__A=None		,						**__A				) ->     str:
								_lowerCAmelCase										=None
								if "feature_extractor" in kwargs:
														warnings.warn(
														    'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
														    ' instead.'		,						__A		,						)
														_lowerCAmelCase										=kwargs.pop('feature_extractor'				)

								_lowerCAmelCase										=image_processor if image_processor is not None else feature_extractor
								if image_processor is None:
														raise ValueError('You need to specify an `image_processor`.'				)
								if tokenizer is None:
														raise ValueError('You need to specify a `tokenizer`.'				)

								super().__init__(__A		,						__A				)






		def __call__(   self		,						__A=None		,						__A=None		,						__A=None		,						**__A				) ->     Optional[int]:

								if text is None and images is None:
														raise ValueError('You have to specify either text or images. Both cannot be none.'				)

								if text is not None:
														_lowerCAmelCase										=self.tokenizer(__A		,						return_tensors=__A		,						**__A				)

								if images is not None:
														_lowerCAmelCase										=self.image_processor(__A		,						return_tensors=__A		,						**__A				)

								if text is not None and images is not None:
														_lowerCAmelCase										=image_features.pixel_values
														return encoding
								elif text is not None:
														return encoding
								else:
														return BatchEncoding(data=dict(**__A				)		,						tensor_type=__A				)






		def 		UpperCamelCase__       (   self		,						*__A		,						**__A				) ->     Any:
								return self.tokenizer.batch_decode(*__A		,						**__A				)






		def 		UpperCamelCase__       (   self		,						*__A		,						**__A				) ->     Optional[int]:
								return self.tokenizer.decode(*__A		,						**__A				)






		@property
		def 		UpperCamelCase__       (   self				) ->     Tuple:
								_lowerCAmelCase										=self.tokenizer.model_input_names
								_lowerCAmelCase										=self.image_processor.model_input_names
								return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names				)				)






		@property
		def 		UpperCamelCase__       (   self				) ->     Optional[int]:
								warnings.warn(
								    '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'		,						__A		,						)
								return self.image_processor_class






		@property
		def 		UpperCamelCase__       (   self				) ->     Optional[Any]:
								warnings.warn(
								    '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'		,						__A		,						)
								return self.image_processor





