

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


SCREAMING_SNAKE_CASE_          =      logging.get_logger(__name__)


if is_vision_available():
					import PIL

class 				a  (	__UpperCAmelCase				):
						_lowercase    						=			["""pixel_values"""]


						def __init__(       self						,					A_ = True						,					A_ = None						,					A_ = PILImageResampling.BICUBIC						,					A_ = True						,					A_ = None						,					A_ = True						,					A_ = 1 / 255						,					A_ = True						,					A_ = None						,					A_ = None						,					A_ = True						,					**A_						,					):







										'''simple docstring'''


										super().__init__(**UpperCAmelCase_	)
										_UpperCAmelCase       :		Tuple 		=    size if size is not None else {"shortest_edge": 224}
										_UpperCAmelCase       :		Tuple 		=    get_size_dict(UpperCAmelCase_						,					default_to_square=UpperCAmelCase_	)
										_UpperCAmelCase       :		Any 		=    crop_size if crop_size is not None else {"height": 224, "width": 224}
										_UpperCAmelCase       :		List[str] 		=    get_size_dict(UpperCAmelCase_						,					default_to_square=UpperCAmelCase_						,					param_name="crop_size"	)

										_UpperCAmelCase       :		List[str] 		=    do_resize
										_UpperCAmelCase       :		Optional[int] 		=    size
										_UpperCAmelCase       :		List[str] 		=    resample
										_UpperCAmelCase       :		Union[str, Any] 		=    do_center_crop
										_UpperCAmelCase       :		Optional[Any] 		=    crop_size
										_UpperCAmelCase       :		List[str] 		=    do_rescale
										_UpperCAmelCase       :		List[str] 		=    rescale_factor
										_UpperCAmelCase       :		int 		=    do_normalize
										_UpperCAmelCase       :		Any 		=    image_mean if image_mean is not None else OPENAI_CLIP_MEAN
										_UpperCAmelCase       :		Union[str, Any] 		=    image_std if image_std is not None else OPENAI_CLIP_STD
										_UpperCAmelCase       :		Union[str, Any] 		=    do_convert_rgb


						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					A_ = PILImageResampling.BICUBIC						,					A_ = None						,					**A_						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    get_size_dict(UpperCAmelCase_						,					default_to_square=UpperCAmelCase_	)
										if "shortest_edge" not in size:
														raise ValueError(f'The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'	)
										_UpperCAmelCase       :		Optional[Any] 		=    get_resize_output_image_size(UpperCAmelCase_						,					size=size["shortest_edge"]						,					default_to_square=UpperCAmelCase_	)
										return resize(UpperCAmelCase_						,					size=UpperCAmelCase_						,					resample=UpperCAmelCase_						,					data_format=UpperCAmelCase_						,					**UpperCAmelCase_	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					A_ = None						,					**A_						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		Dict 		=    get_size_dict(UpperCAmelCase_	)
										if "height" not in size or "width" not in size:
														raise ValueError(f'The `size` parameter must contain the keys (height, width). Got {size.keys()}'	)
										return center_crop(UpperCAmelCase_						,					size=(size["height"], size["width"])						,					data_format=UpperCAmelCase_						,					**UpperCAmelCase_	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					A_ = None						,					**A_						,					):







										'''simple docstring'''


										return rescale(UpperCAmelCase_						,					scale=UpperCAmelCase_						,					data_format=UpperCAmelCase_						,					**UpperCAmelCase_	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					A_						,					A_ = None						,					**A_						,					):







										'''simple docstring'''


										return normalize(UpperCAmelCase_						,					mean=UpperCAmelCase_						,					std=UpperCAmelCase_						,					data_format=UpperCAmelCase_						,					**UpperCAmelCase_	)





						def        _UpperCAmelCase				(       self						,					A_						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = ChannelDimension.FIRST						,					**A_						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[Any] 		=    do_resize if do_resize is not None else self.do_resize
										_UpperCAmelCase       :		Tuple 		=    size if size is not None else self.size
										_UpperCAmelCase       :		List[str] 		=    get_size_dict(UpperCAmelCase_						,					param_name="size"						,					default_to_square=UpperCAmelCase_	)
										_UpperCAmelCase       :		Dict 		=    resample if resample is not None else self.resample
										_UpperCAmelCase       :		Any 		=    do_center_crop if do_center_crop is not None else self.do_center_crop
										_UpperCAmelCase       :		List[str] 		=    crop_size if crop_size is not None else self.crop_size
										_UpperCAmelCase       :		Tuple 		=    get_size_dict(UpperCAmelCase_						,					param_name="crop_size"						,					default_to_square=UpperCAmelCase_	)
										_UpperCAmelCase       :		List[Any] 		=    do_rescale if do_rescale is not None else self.do_rescale
										_UpperCAmelCase       :		Any 		=    rescale_factor if rescale_factor is not None else self.rescale_factor
										_UpperCAmelCase       :		Optional[int] 		=    do_normalize if do_normalize is not None else self.do_normalize
										_UpperCAmelCase       :		str 		=    image_mean if image_mean is not None else self.image_mean
										_UpperCAmelCase       :		Any 		=    image_std if image_std is not None else self.image_std
										_UpperCAmelCase       :		Union[str, Any] 		=    do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

										_UpperCAmelCase       :		int 		=    make_list_of_images(UpperCAmelCase_	)

										if not valid_images(UpperCAmelCase_	):
														raise ValueError(
														    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
														    "torch.Tensor, tf.Tensor or jax.ndarray."	)

										if do_resize and size is None:
														raise ValueError("Size must be specified if do_resize is True."	)

										if do_center_crop and crop_size is None:
														raise ValueError("Crop size must be specified if do_center_crop is True."	)

										if do_rescale and rescale_factor is None:
														raise ValueError("Rescale factor must be specified if do_rescale is True."	)

										if do_normalize and (image_mean is None or image_std is None):
														raise ValueError("Image mean and std must be specified if do_normalize is True."	)

										# PIL RGBA images are converted to RGB
										if do_convert_rgb:
														_UpperCAmelCase       :		Tuple 		=    [convert_to_rgb(UpperCAmelCase_	) for image in images]

										# All transformations expect numpy arrays.
										_UpperCAmelCase       :		str 		=    [to_numpy_array(UpperCAmelCase_	) for image in images]

										if do_resize:
														_UpperCAmelCase       :		List[Any] 		=    [self.resize(image=UpperCAmelCase_						,					size=UpperCAmelCase_						,					resample=UpperCAmelCase_	) for image in images]

										if do_center_crop:
														_UpperCAmelCase       :		Dict 		=    [self.center_crop(image=UpperCAmelCase_						,					size=UpperCAmelCase_	) for image in images]

										if do_rescale:
														_UpperCAmelCase       :		List[Any] 		=    [self.rescale(image=UpperCAmelCase_						,					scale=UpperCAmelCase_	) for image in images]

										if do_normalize:
														_UpperCAmelCase       :		Union[str, Any] 		=    [self.normalize(image=UpperCAmelCase_						,					mean=UpperCAmelCase_						,					std=UpperCAmelCase_	) for image in images]

										_UpperCAmelCase       :		Optional[int] 		=    [to_channel_dimension_format(UpperCAmelCase_						,					UpperCAmelCase_	) for image in images]

										_UpperCAmelCase       :		Dict 		=    {"pixel_values": images}
										return BatchFeature(data=UpperCAmelCase_						,					tensor_type=UpperCAmelCase_	)





import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY

@is_pipeline_test
class 				a  (	unittest.TestCase				):
						_lowercase    						=			MODEL_FOR_MASKED_LM_MAPPING
						_lowercase    						=			TF_MODEL_FOR_MASKED_LM_MAPPING


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										super().tearDown()
										# clean-up as much as possible GPU memory occupied by PyTorch
										gc.collect()
										if is_torch_available():
														import torch

														torch.cuda.empty_cache()


						@require_tf
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					top_k=2						,					framework="tf"	)
										_UpperCAmelCase       :		Union[str, Any] 		=    unmasker("My name is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is grouped", "score": 2.1e-05, "token": 38015, "token_str": " grouped"},
										        {"sequence": "My name is accuser", "score": 2.1e-05, "token": 25506, "token_str": " accuser"},
										    ]						,					)

										_UpperCAmelCase       :		Optional[int] 		=    unmasker("The largest city in France is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {
										            "sequence": "The largest city in France is grouped",
										            "score": 2.1e-05,
										            "token": 38015,
										            "token_str": " grouped",
										        },
										        {
										            "sequence": "The largest city in France is accuser",
										            "score": 2.1e-05,
										            "token": 25506,
										            "token_str": " accuser",
										        },
										    ]						,					)

										_UpperCAmelCase       :		Optional[Any] 		=    unmasker("My name is <mask>"						,					targets=[" Patrick", " Clara", " Teven"]						,					top_k=3	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
										        {"sequence": "My name is Patrick", "score": 2e-05, "token": 3499, "token_str": " Patrick"},
										        {"sequence": "My name is Te", "score": 1.9e-05, "token": 2941, "token_str": " Te"},
										    ]						,					)


						@require_torch
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[Any] 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					top_k=2						,					framework="pt"	)

										_UpperCAmelCase       :		List[str] 		=    unmasker("My name is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is Maul", "score": 2.2e-05, "token": 35676, "token_str": " Maul"},
										        {"sequence": "My name isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
										    ]						,					)

										_UpperCAmelCase       :		str 		=    unmasker("The largest city in France is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {
										            "sequence": "The largest city in France is Maul",
										            "score": 2.2e-05,
										            "token": 35676,
										            "token_str": " Maul",
										        },
										        {"sequence": "The largest city in France isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
										    ]						,					)

										_UpperCAmelCase       :		List[Any] 		=    unmasker("My name is <mask>"						,					targets=[" Patrick", " Clara", " Teven"]						,					top_k=3	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is Patrick", "score": 2.1e-05, "token": 3499, "token_str": " Patrick"},
										        {"sequence": "My name is Te", "score": 2e-05, "token": 2941, "token_str": " Te"},
										        {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
										    ]						,					)

										_UpperCAmelCase       :		Optional[Any] 		=    unmasker("My name is <mask> <mask>"						,					top_k=2	)

										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        [
										            {
										                "score": 2.2e-05,
										                "token": 35676,
										                "token_str": " Maul",
										                "sequence": "<s>My name is Maul<mask></s>",
										            },
										            {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
										        ],
										        [
										            {
										                "score": 2.2e-05,
										                "token": 35676,
										                "token_str": " Maul",
										                "sequence": "<s>My name is<mask> Maul</s>",
										            },
										            {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
										        ],
										    ]						,					)


						@require_torch_gpu
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    pipeline("fill-mask"						,					model="hf-internal-testing/tiny-random-distilbert"						,					device=0						,					framework="pt"	)

										# convert model to fp16
										pipe.model.half()

										_UpperCAmelCase       :		int 		=    pipe("Paris is the [MASK] of France."	)
										# We actually don't care about the result, we just want to make sure
										# it works, meaning the float16 tensor got casted back to float32
										# for postprocessing.
										self.assertIsInstance(A_						,					A_	)


						@slow
						@require_torch
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    pipeline(task="fill-mask"						,					model="distilroberta-base"						,					top_k=2						,					framework="pt"	)
										self.run_large_test(A_	)


						@slow
						@require_tf
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    pipeline(task="fill-mask"						,					model="distilroberta-base"						,					top_k=2						,					framework="tf"	)
										self.run_large_test(A_	)


						def        _UpperCAmelCase				(       self						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    unmasker("My name is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_	)						,					[
										        {"sequence": "My name is John", "score": 0.0_08, "token": 610, "token_str": " John"},
										        {"sequence": "My name is Chris", "score": 0.0_07, "token": 1573, "token_str": " Chris"},
										    ]						,					)
										_UpperCAmelCase       :		Optional[int] 		=    unmasker("The largest city in France is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_	)						,					[
										        {
										            "sequence": "The largest city in France is Paris",
										            "score": 0.2_51,
										            "token": 2201,
										            "token_str": " Paris",
										        },
										        {
										            "sequence": "The largest city in France is Lyon",
										            "score": 0.2_14,
										            "token": 12790,
										            "token_str": " Lyon",
										        },
										    ]						,					)

										_UpperCAmelCase       :		Any 		=    unmasker("My name is <mask>"						,					targets=[" Patrick", " Clara", " Teven"]						,					top_k=3	)
										self.assertEqual(
										    nested_simplify(A_	)						,					[
										        {"sequence": "My name is Patrick", "score": 0.0_05, "token": 3499, "token_str": " Patrick"},
										        {"sequence": "My name is Clara", "score": 0.0_00, "token": 13606, "token_str": " Clara"},
										        {"sequence": "My name is Te", "score": 0.0_00, "token": 2941, "token_str": " Te"},
										    ]						,					)


						@require_torch
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					framework="pt"	)
										_UpperCAmelCase       :		Optional[int] 		=    None
										_UpperCAmelCase       :		List[Any] 		=    None
										self.run_pipeline_test(A_						,					[]	)


						@require_tf
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					framework="tf"	)
										_UpperCAmelCase       :		Union[str, Any] 		=    None
										_UpperCAmelCase       :		str 		=    None
										self.run_pipeline_test(A_						,					[]	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					A_	):







										'''simple docstring'''


										if tokenizer is None or tokenizer.mask_token_id is None:
														self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)"	)

										_UpperCAmelCase       :		Tuple 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		Tuple 		=    [
										    f'This is another {tokenizer.mask_token} test',
										]
										return fill_masker, examples


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Dict 		=    fill_masker.tokenizer
										_UpperCAmelCase       :		Tuple 		=    fill_masker.model

										_UpperCAmelCase       :		Union[str, Any] 		=    fill_masker(
										    f'This is a {tokenizer.mask_token}'						,					)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)

										_UpperCAmelCase       :		Optional[Any] 		=    fill_masker([f'This is a {tokenizer.mask_token}']	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)

										_UpperCAmelCase       :		List[str] 		=    fill_masker([f'This is a {tokenizer.mask_token}', f'Another {tokenizer.mask_token} great test.']	)
										self.assertEqual(
										    A_						,					[
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										    ]						,					)

										with self.assertRaises(A_	):
														fill_masker([None]	)
										# No mask_token is not supported
										with self.assertRaises(A_	):
														fill_masker("This is"	)

										self.run_test_top_k(A_						,					A_	)
										self.run_test_targets(A_						,					A_	)
										self.run_test_top_k_targets(A_						,					A_	)
										self.fill_mask_with_duplicate_targets_and_top_k(A_						,					A_	)
										self.fill_mask_with_multiple_masks(A_						,					A_	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    tokenizer.get_vocab()
										_UpperCAmelCase       :		int 		=    sorted(vocab.keys()	)[:2]
										# Pipeline argument
										_UpperCAmelCase       :		Any 		=    FillMaskPipeline(model=A_						,					tokenizer=A_						,					targets=A_	)
										_UpperCAmelCase       :		Optional[int] 		=    fill_masker(f'This is a {tokenizer.mask_token}'	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)
										_UpperCAmelCase       :		Tuple 		=    {vocab[el] for el in targets}
										self.assertEqual({el["token"] for el in outputs}						,					A_	)
										_UpperCAmelCase       :		List[Any] 		=    [tokenizer.decode([x]	) for x in target_ids]
										self.assertEqual({el["token_str"] for el in outputs}						,					set(A_	)	)

										# Call argument
										_UpperCAmelCase       :		Union[str, Any] 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		List[Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=A_	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)
										_UpperCAmelCase       :		Optional[Any] 		=    {vocab[el] for el in targets}
										self.assertEqual({el["token"] for el in outputs}						,					A_	)
										_UpperCAmelCase       :		Any 		=    [tokenizer.decode([x]	) for x in target_ids]
										self.assertEqual({el["token_str"] for el in outputs}						,					set(A_	)	)

										# Score equivalence
										_UpperCAmelCase       :		int 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=A_	)
										_UpperCAmelCase       :		Tuple 		=    [top_mask["token_str"] for top_mask in outputs]
										_UpperCAmelCase       :		Union[str, Any] 		=    [top_mask["score"] for top_mask in outputs]

										# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
										if set(A_	) == set(A_	):
														_UpperCAmelCase       :		List[str] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=A_	)
														_UpperCAmelCase       :		List[str] 		=    [top_mask["score"] for top_mask in unmasked_targets]
														self.assertEqual(nested_simplify(A_	)						,					nested_simplify(A_	)	)

										# Raises with invalid
										with self.assertRaises(A_	):
														_UpperCAmelCase       :		Optional[int] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=[]	)
										# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
										if "" not in tokenizer.get_vocab():
														with self.assertRaises(A_	):
																		_UpperCAmelCase       :		Dict 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=[""]	)
														with self.assertRaises(A_	):
																		_UpperCAmelCase       :		List[str] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=""	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    FillMaskPipeline(model=A_						,					tokenizer=A_						,					top_k=2	)
										_UpperCAmelCase       :		Union[str, Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)

										_UpperCAmelCase       :		int 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		Optional[Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					top_k=2	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)
										self.assertEqual(nested_simplify(A_	)						,					nested_simplify(A_	)	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    tokenizer.get_vocab()
										_UpperCAmelCase       :		Union[str, Any] 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)

										# top_k=2, ntargets=3
										_UpperCAmelCase       :		Tuple 		=    sorted(vocab.keys()	)[:3]
										_UpperCAmelCase       :		Union[str, Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					top_k=2						,					targets=A_	)

										# If we use the most probably targets, and filter differently, we should still
										# have the same results
										_UpperCAmelCase       :		Tuple 		=    [el["token_str"] for el in sorted(A_						,					key=lambda A_	: x["score"]						,					reverse=A_	)]
										# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
										if set(A_	).issubset(A_	):
														_UpperCAmelCase       :		int 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					top_k=3						,					targets=A_	)
														# They should yield exactly the same result
														self.assertEqual(nested_simplify(A_	)						,					nested_simplify(A_	)	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		Dict 		=    tokenizer.get_vocab()
										# String duplicates + id duplicates
										_UpperCAmelCase       :		Dict 		=    sorted(vocab.keys()	)[:3]
										_UpperCAmelCase       :		Tuple 		=    [targets[0], targets[1], targets[0], targets[2], targets[1]]
										_UpperCAmelCase       :		Dict 		=    fill_masker(f'My name is {tokenizer.mask_token}'						,					targets=A_						,					top_k=10	)

										# The target list contains duplicates, so we can't output more
										# than them
										self.assertEqual(len(A_	)						,					3	)





						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)

										_UpperCAmelCase       :		List[str] 		=    fill_masker(
										    f'This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'						,					top_k=2	)
										self.assertEqual(
										    A_						,					[
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										    ]						,					)


