

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_albert import AlbertTokenizer
else:
					SCREAMING_SNAKE_CASE_          =      None

SCREAMING_SNAKE_CASE_          =      logging.get_logger(__name__)
SCREAMING_SNAKE_CASE_          =      {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_          =      {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE_          =      {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

SCREAMING_SNAKE_CASE_          =      '▁'

class 				a  (	UpperCAmelCase				):
						_lowercase    						=			VOCAB_FILES_NAMES
						_lowercase    						=			PRETRAINED_VOCAB_FILES_MAP
						_lowercase    						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowercase    						=			AlbertTokenizer


						def __init__(       self						,					A_=None						,					A_=None						,					A_=True						,					A_=True						,					A_=False						,					A_="[CLS]"						,					A_="[SEP]"						,					A_="<unk>"						,					A_="[SEP]"						,					A_="<pad>"						,					A_="[CLS]"						,					A_="[MASK]"						,					**A_						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    (
										    AddedToken(__A						,					lstrip=__A						,					rstrip=__A						,					normalized=__A	)
										    if isinstance(__A						,					__A	)
										    else mask_token
										)

										super().__init__(
										    __A						,					tokenizer_file=__A						,					do_lower_case=__A						,					remove_space=__A						,					keep_accents=__A						,					bos_token=__A						,					eos_token=__A						,					unk_token=__A						,					sep_token=__A						,					pad_token=__A						,					cls_token=__A						,					mask_token=__A						,					**__A						,					)

										_UpperCAmelCase       :		Dict 		=    do_lower_case
										_UpperCAmelCase       :		List[str] 		=    remove_space
										_UpperCAmelCase       :		Any 		=    keep_accents
										_UpperCAmelCase       :		Tuple 		=    vocab_file
										_UpperCAmelCase       :		Optional[int] 		=    False if not self.vocab_file else True


						def        _UpperCAmelCase				(       self						,					A_						,					A_ = None	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    [self.sep_token_id]
										_UpperCAmelCase       :		Any 		=    [self.cls_token_id]
										if token_ids_a is None:
														return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep


						def        _UpperCAmelCase				(       self						,					A_						,					A_ = None	):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[Any] 		=    [self.sep_token_id]
										_UpperCAmelCase       :		Union[str, Any] 		=    [self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep	) * [0]
										return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





						def        _UpperCAmelCase				(       self						,					A_						,					A_ = None	):







										'''simple docstring'''


										if not self.can_save_slow_tokenizer:
														raise ValueError(
														    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
														    "tokenizer."	)

										if not os.path.isdir(__A	):
														logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
														return
										_UpperCAmelCase       :		Union[str, Any] 		=    os.path.join(
										    __A						,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

										if os.path.abspath(self.vocab_file	) != os.path.abspath(__A	):
														copyfile(self.vocab_file						,					__A	)

										return (out_vocab_file,)





import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY

@is_pipeline_test
class 				a  (	unittest.TestCase				):
						_lowercase    						=			MODEL_FOR_MASKED_LM_MAPPING
						_lowercase    						=			TF_MODEL_FOR_MASKED_LM_MAPPING


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										super().tearDown()
										# clean-up as much as possible GPU memory occupied by PyTorch
										gc.collect()
										if is_torch_available():
														import torch

														torch.cuda.empty_cache()


						@require_tf
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					top_k=2						,					framework="tf"	)
										_UpperCAmelCase       :		Union[str, Any] 		=    unmasker("My name is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is grouped", "score": 2.1e-05, "token": 38015, "token_str": " grouped"},
										        {"sequence": "My name is accuser", "score": 2.1e-05, "token": 25506, "token_str": " accuser"},
										    ]						,					)

										_UpperCAmelCase       :		Optional[int] 		=    unmasker("The largest city in France is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {
										            "sequence": "The largest city in France is grouped",
										            "score": 2.1e-05,
										            "token": 38015,
										            "token_str": " grouped",
										        },
										        {
										            "sequence": "The largest city in France is accuser",
										            "score": 2.1e-05,
										            "token": 25506,
										            "token_str": " accuser",
										        },
										    ]						,					)

										_UpperCAmelCase       :		Optional[Any] 		=    unmasker("My name is <mask>"						,					targets=[" Patrick", " Clara", " Teven"]						,					top_k=3	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
										        {"sequence": "My name is Patrick", "score": 2e-05, "token": 3499, "token_str": " Patrick"},
										        {"sequence": "My name is Te", "score": 1.9e-05, "token": 2941, "token_str": " Te"},
										    ]						,					)


						@require_torch
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[Any] 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					top_k=2						,					framework="pt"	)

										_UpperCAmelCase       :		List[str] 		=    unmasker("My name is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is Maul", "score": 2.2e-05, "token": 35676, "token_str": " Maul"},
										        {"sequence": "My name isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
										    ]						,					)

										_UpperCAmelCase       :		str 		=    unmasker("The largest city in France is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {
										            "sequence": "The largest city in France is Maul",
										            "score": 2.2e-05,
										            "token": 35676,
										            "token_str": " Maul",
										        },
										        {"sequence": "The largest city in France isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
										    ]						,					)

										_UpperCAmelCase       :		List[Any] 		=    unmasker("My name is <mask>"						,					targets=[" Patrick", " Clara", " Teven"]						,					top_k=3	)
										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        {"sequence": "My name is Patrick", "score": 2.1e-05, "token": 3499, "token_str": " Patrick"},
										        {"sequence": "My name is Te", "score": 2e-05, "token": 2941, "token_str": " Te"},
										        {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
										    ]						,					)

										_UpperCAmelCase       :		Optional[Any] 		=    unmasker("My name is <mask> <mask>"						,					top_k=2	)

										self.assertEqual(
										    nested_simplify(A_						,					decimals=6	)						,					[
										        [
										            {
										                "score": 2.2e-05,
										                "token": 35676,
										                "token_str": " Maul",
										                "sequence": "<s>My name is Maul<mask></s>",
										            },
										            {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
										        ],
										        [
										            {
										                "score": 2.2e-05,
										                "token": 35676,
										                "token_str": " Maul",
										                "sequence": "<s>My name is<mask> Maul</s>",
										            },
										            {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
										        ],
										    ]						,					)


						@require_torch_gpu
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    pipeline("fill-mask"						,					model="hf-internal-testing/tiny-random-distilbert"						,					device=0						,					framework="pt"	)

										# convert model to fp16
										pipe.model.half()

										_UpperCAmelCase       :		int 		=    pipe("Paris is the [MASK] of France."	)
										# We actually don't care about the result, we just want to make sure
										# it works, meaning the float16 tensor got casted back to float32
										# for postprocessing.
										self.assertIsInstance(A_						,					A_	)


						@slow
						@require_torch
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    pipeline(task="fill-mask"						,					model="distilroberta-base"						,					top_k=2						,					framework="pt"	)
										self.run_large_test(A_	)


						@slow
						@require_tf
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    pipeline(task="fill-mask"						,					model="distilroberta-base"						,					top_k=2						,					framework="tf"	)
										self.run_large_test(A_	)


						def        _UpperCAmelCase				(       self						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    unmasker("My name is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_	)						,					[
										        {"sequence": "My name is John", "score": 0.0_08, "token": 610, "token_str": " John"},
										        {"sequence": "My name is Chris", "score": 0.0_07, "token": 1573, "token_str": " Chris"},
										    ]						,					)
										_UpperCAmelCase       :		Optional[int] 		=    unmasker("The largest city in France is <mask>"	)
										self.assertEqual(
										    nested_simplify(A_	)						,					[
										        {
										            "sequence": "The largest city in France is Paris",
										            "score": 0.2_51,
										            "token": 2201,
										            "token_str": " Paris",
										        },
										        {
										            "sequence": "The largest city in France is Lyon",
										            "score": 0.2_14,
										            "token": 12790,
										            "token_str": " Lyon",
										        },
										    ]						,					)

										_UpperCAmelCase       :		Any 		=    unmasker("My name is <mask>"						,					targets=[" Patrick", " Clara", " Teven"]						,					top_k=3	)
										self.assertEqual(
										    nested_simplify(A_	)						,					[
										        {"sequence": "My name is Patrick", "score": 0.0_05, "token": 3499, "token_str": " Patrick"},
										        {"sequence": "My name is Clara", "score": 0.0_00, "token": 13606, "token_str": " Clara"},
										        {"sequence": "My name is Te", "score": 0.0_00, "token": 2941, "token_str": " Te"},
										    ]						,					)


						@require_torch
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					framework="pt"	)
										_UpperCAmelCase       :		Optional[int] 		=    None
										_UpperCAmelCase       :		List[Any] 		=    None
										self.run_pipeline_test(A_						,					[]	)


						@require_tf
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    pipeline(task="fill-mask"						,					model="sshleifer/tiny-distilroberta-base"						,					framework="tf"	)
										_UpperCAmelCase       :		Union[str, Any] 		=    None
										_UpperCAmelCase       :		str 		=    None
										self.run_pipeline_test(A_						,					[]	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					A_	):







										'''simple docstring'''


										if tokenizer is None or tokenizer.mask_token_id is None:
														self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)"	)

										_UpperCAmelCase       :		Tuple 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		Tuple 		=    [
										    f'This is another {tokenizer.mask_token} test',
										]
										return fill_masker, examples


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Dict 		=    fill_masker.tokenizer
										_UpperCAmelCase       :		Tuple 		=    fill_masker.model

										_UpperCAmelCase       :		Union[str, Any] 		=    fill_masker(
										    f'This is a {tokenizer.mask_token}'						,					)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)

										_UpperCAmelCase       :		Optional[Any] 		=    fill_masker([f'This is a {tokenizer.mask_token}']	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)

										_UpperCAmelCase       :		List[str] 		=    fill_masker([f'This is a {tokenizer.mask_token}', f'Another {tokenizer.mask_token} great test.']	)
										self.assertEqual(
										    A_						,					[
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										    ]						,					)

										with self.assertRaises(A_	):
														fill_masker([None]	)
										# No mask_token is not supported
										with self.assertRaises(A_	):
														fill_masker("This is"	)

										self.run_test_top_k(A_						,					A_	)
										self.run_test_targets(A_						,					A_	)
										self.run_test_top_k_targets(A_						,					A_	)
										self.fill_mask_with_duplicate_targets_and_top_k(A_						,					A_	)
										self.fill_mask_with_multiple_masks(A_						,					A_	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    tokenizer.get_vocab()
										_UpperCAmelCase       :		int 		=    sorted(vocab.keys()	)[:2]
										# Pipeline argument
										_UpperCAmelCase       :		Any 		=    FillMaskPipeline(model=A_						,					tokenizer=A_						,					targets=A_	)
										_UpperCAmelCase       :		Optional[int] 		=    fill_masker(f'This is a {tokenizer.mask_token}'	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)
										_UpperCAmelCase       :		Tuple 		=    {vocab[el] for el in targets}
										self.assertEqual({el["token"] for el in outputs}						,					A_	)
										_UpperCAmelCase       :		List[Any] 		=    [tokenizer.decode([x]	) for x in target_ids]
										self.assertEqual({el["token_str"] for el in outputs}						,					set(A_	)	)

										# Call argument
										_UpperCAmelCase       :		Union[str, Any] 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		List[Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=A_	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)
										_UpperCAmelCase       :		Optional[Any] 		=    {vocab[el] for el in targets}
										self.assertEqual({el["token"] for el in outputs}						,					A_	)
										_UpperCAmelCase       :		Any 		=    [tokenizer.decode([x]	) for x in target_ids]
										self.assertEqual({el["token_str"] for el in outputs}						,					set(A_	)	)

										# Score equivalence
										_UpperCAmelCase       :		int 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=A_	)
										_UpperCAmelCase       :		Tuple 		=    [top_mask["token_str"] for top_mask in outputs]
										_UpperCAmelCase       :		Union[str, Any] 		=    [top_mask["score"] for top_mask in outputs]

										# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
										if set(A_	) == set(A_	):
														_UpperCAmelCase       :		List[str] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=A_	)
														_UpperCAmelCase       :		List[str] 		=    [top_mask["score"] for top_mask in unmasked_targets]
														self.assertEqual(nested_simplify(A_	)						,					nested_simplify(A_	)	)

										# Raises with invalid
										with self.assertRaises(A_	):
														_UpperCAmelCase       :		Optional[int] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=[]	)
										# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
										if "" not in tokenizer.get_vocab():
														with self.assertRaises(A_	):
																		_UpperCAmelCase       :		Dict 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=[""]	)
														with self.assertRaises(A_	):
																		_UpperCAmelCase       :		List[str] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					targets=""	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    FillMaskPipeline(model=A_						,					tokenizer=A_						,					top_k=2	)
										_UpperCAmelCase       :		Union[str, Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)

										_UpperCAmelCase       :		int 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		Optional[Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					top_k=2	)
										self.assertEqual(
										    A_						,					[
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										    ]						,					)
										self.assertEqual(nested_simplify(A_	)						,					nested_simplify(A_	)	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    tokenizer.get_vocab()
										_UpperCAmelCase       :		Union[str, Any] 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)

										# top_k=2, ntargets=3
										_UpperCAmelCase       :		Tuple 		=    sorted(vocab.keys()	)[:3]
										_UpperCAmelCase       :		Union[str, Any] 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					top_k=2						,					targets=A_	)

										# If we use the most probably targets, and filter differently, we should still
										# have the same results
										_UpperCAmelCase       :		Tuple 		=    [el["token_str"] for el in sorted(A_						,					key=lambda A_	: x["score"]						,					reverse=A_	)]
										# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
										if set(A_	).issubset(A_	):
														_UpperCAmelCase       :		int 		=    fill_masker(f'This is a {tokenizer.mask_token}'						,					top_k=3						,					targets=A_	)
														# They should yield exactly the same result
														self.assertEqual(nested_simplify(A_	)						,					nested_simplify(A_	)	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)
										_UpperCAmelCase       :		Dict 		=    tokenizer.get_vocab()
										# String duplicates + id duplicates
										_UpperCAmelCase       :		Dict 		=    sorted(vocab.keys()	)[:3]
										_UpperCAmelCase       :		Tuple 		=    [targets[0], targets[1], targets[0], targets[2], targets[1]]
										_UpperCAmelCase       :		Dict 		=    fill_masker(f'My name is {tokenizer.mask_token}'						,					targets=A_						,					top_k=10	)

										# The target list contains duplicates, so we can't output more
										# than them
										self.assertEqual(len(A_	)						,					3	)





						def        _UpperCAmelCase				(       self						,					A_						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    FillMaskPipeline(model=A_						,					tokenizer=A_	)

										_UpperCAmelCase       :		List[str] 		=    fill_masker(
										    f'This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'						,					top_k=2	)
										self.assertEqual(
										    A_						,					[
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										        [
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										            {"sequence": ANY(A_	), "score": ANY(A_	), "token": ANY(A_	), "token_str": ANY(A_	)},
										        ],
										    ]						,					)


