"""simple docstring"""







import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY







@is_pipeline_test
class  UpperCamelCase	(    unittest.TestCase						):
   lowercase							=						MODEL_FOR_MASKED_LM_MAPPING
   lowercase							=						TF_MODEL_FOR_MASKED_LM_MAPPING
   def     _UpperCAmelCase     (    self    )			->   List[str]:


        '''simple docstring'''


        super().tearDown()
        # clean-up as much as possible GPU memory occupied by PyTorch
        gc.collect()
        if is_torch_available():
             import torch

             torch.cuda.empty_cache()
   @require_tf
   def     _UpperCAmelCase     (    self    )			->   List[Any]:


        '''simple docstring'''


        lowercase_			:       List[str]									=			pipeline(task='fill-mask'  ,model='sshleifer/tiny-distilroberta-base'  ,top_k=2  ,framework='tf'    )
        lowercase_			:       List[str]									=			unmasker('My name is <mask>'    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase  ,decimals=6    )  ,[
                {'sequence': 'My name is grouped', 'score': 2.1e-05, 'token': 3_8015, 'token_str': ' grouped'},
                {'sequence': 'My name is accuser', 'score': 2.1e-05, 'token': 2_5506, 'token_str': ' accuser'},
            ]  ,)

        lowercase_			:       Optional[Any]									=			unmasker('The largest city in France is <mask>'    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase  ,decimals=6    )  ,[
                {
                    'sequence': 'The largest city in France is grouped',
                    'score': 2.1e-05,
                    'token': 3_8015,
                    'token_str': ' grouped',
                },
                {
                    'sequence': 'The largest city in France is accuser',
                    'score': 2.1e-05,
                    'token': 2_5506,
                    'token_str': ' accuser',
                },
            ]  ,)

        lowercase_			:       str									=			unmasker('My name is <mask>'  ,targets=[' Patrick', ' Clara', ' Teven']  ,top_k=3    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase  ,decimals=6    )  ,[
                {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 1_3606, 'token_str': ' Clara'},
                {'sequence': 'My name is Patrick', 'score': 2e-05, 'token': 3499, 'token_str': ' Patrick'},
                {'sequence': 'My name is Te', 'score': 1.9e-05, 'token': 2941, 'token_str': ' Te'},
            ]  ,)
   @require_torch
   def     _UpperCAmelCase     (    self    )			->   Optional[Any]:


        '''simple docstring'''


        lowercase_			:       Dict									=			pipeline(task='fill-mask'  ,model='sshleifer/tiny-distilroberta-base'  ,top_k=2  ,framework='pt'    )

        lowercase_			:       Optional[int]									=			unmasker('My name is <mask>'    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase  ,decimals=6    )  ,[
                {'sequence': 'My name is Maul', 'score': 2.2e-05, 'token': 3_5676, 'token_str': ' Maul'},
                {'sequence': 'My name isELS', 'score': 2.2e-05, 'token': 1_6416, 'token_str': 'ELS'},
            ]  ,)

        lowercase_			:       str									=			unmasker('The largest city in France is <mask>'    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase  ,decimals=6    )  ,[
                {
                    'sequence': 'The largest city in France is Maul',
                    'score': 2.2e-05,
                    'token': 3_5676,
                    'token_str': ' Maul',
                },
                {'sequence': 'The largest city in France isELS', 'score': 2.2e-05, 'token': 1_6416, 'token_str': 'ELS'},
            ]  ,)

        lowercase_			:       str									=			unmasker('My name is <mask>'  ,targets=[' Patrick', ' Clara', ' Teven']  ,top_k=3    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase  ,decimals=6    )  ,[
                {'sequence': 'My name is Patrick', 'score': 2.1e-05, 'token': 3499, 'token_str': ' Patrick'},
                {'sequence': 'My name is Te', 'score': 2e-05, 'token': 2941, 'token_str': ' Te'},
                {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 1_3606, 'token_str': ' Clara'},
            ]  ,)

        lowercase_			:       str									=			unmasker('My name is <mask> <mask>'  ,top_k=2    )

        self.assertEqual(
            nested_simplify(__UpperCamelCase  ,decimals=6    )  ,[
                [
                    {
                        'score': 2.2e-05,
                        'token': 3_5676,
                        'token_str': ' Maul',
                        'sequence': '<s>My name is Maul<mask></s>',
                    },
                    {'score': 2.2e-05, 'token': 1_6416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
                ],
                [
                    {
                        'score': 2.2e-05,
                        'token': 3_5676,
                        'token_str': ' Maul',
                        'sequence': '<s>My name is<mask> Maul</s>',
                    },
                    {'score': 2.2e-05, 'token': 1_6416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
                ],
            ]  ,)
   @require_torch_gpu
   def     _UpperCAmelCase     (    self    )			->   str:


        '''simple docstring'''


        lowercase_			:       Tuple									=			pipeline('fill-mask'  ,model='hf-internal-testing/tiny-random-distilbert'  ,device=0  ,framework='pt'    )

        # convert model to fp16
        pipe.model.half()

        lowercase_			:       Dict									=			pipe('Paris is the [MASK] of France.'    )
        # We actually don't care about the result, we just want to make sure
        # it works, meaning the float16 tensor got casted back to float32
        # for postprocessing.
        self.assertIsInstance(__UpperCamelCase  ,__UpperCamelCase    )
   @slow
   @require_torch
   def     _UpperCAmelCase     (    self    )			->   int:


        '''simple docstring'''


        lowercase_			:       Optional[Any]									=			pipeline(task='fill-mask'  ,model='distilroberta-base'  ,top_k=2  ,framework='pt'    )
        self.run_large_test(__UpperCamelCase    )
   @slow
   @require_tf
   def     _UpperCAmelCase     (    self    )			->   Union[str, Any]:


        '''simple docstring'''


        lowercase_			:       Optional[Any]									=			pipeline(task='fill-mask'  ,model='distilroberta-base'  ,top_k=2  ,framework='tf'    )
        self.run_large_test(__UpperCamelCase    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   int:


        '''simple docstring'''


        lowercase_			:       List[Any]									=			unmasker('My name is <mask>'    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase    )  ,[
                {'sequence': 'My name is John', 'score': 0.008, 'token': 610, 'token_str': ' John'},
                {'sequence': 'My name is Chris', 'score': 0.007, 'token': 1573, 'token_str': ' Chris'},
            ]  ,)
        lowercase_			:       Any									=			unmasker('The largest city in France is <mask>'    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase    )  ,[
                {
                    'sequence': 'The largest city in France is Paris',
                    'score': 0.251,
                    'token': 2201,
                    'token_str': ' Paris',
                },
                {
                    'sequence': 'The largest city in France is Lyon',
                    'score': 0.214,
                    'token': 1_2790,
                    'token_str': ' Lyon',
                },
            ]  ,)

        lowercase_			:       Union[str, Any]									=			unmasker('My name is <mask>'  ,targets=[' Patrick', ' Clara', ' Teven']  ,top_k=3    )
        self.assertEqual(
            nested_simplify(__UpperCamelCase    )  ,[
                {'sequence': 'My name is Patrick', 'score': 0.005, 'token': 3499, 'token_str': ' Patrick'},
                {'sequence': 'My name is Clara', 'score': 0.000, 'token': 1_3606, 'token_str': ' Clara'},
                {'sequence': 'My name is Te', 'score': 0.000, 'token': 2941, 'token_str': ' Te'},
            ]  ,)
   @require_torch
   def     _UpperCAmelCase     (    self    )			->   List[str]:


        '''simple docstring'''


        lowercase_			:       Optional[int]									=			pipeline(task='fill-mask'  ,model='sshleifer/tiny-distilroberta-base'  ,framework='pt'    )
        lowercase_			:       int									=			None
        lowercase_			:       Optional[int]									=			None
        self.run_pipeline_test(__UpperCamelCase  ,[]    )
   @require_tf
   def     _UpperCAmelCase     (    self    )			->   Tuple:


        '''simple docstring'''


        lowercase_			:       List[str]									=			pipeline(task='fill-mask'  ,model='sshleifer/tiny-distilroberta-base'  ,framework='tf'    )
        lowercase_			:       Dict									=			None
        lowercase_			:       Any									=			None
        self.run_pipeline_test(__UpperCamelCase  ,[]    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Optional[int]:


        '''simple docstring'''


        if tokenizer is None or tokenizer.mask_token_id is None:
             self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'    )

        lowercase_			:       Union[str, Any]									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase    )
        lowercase_			:       Union[str, Any]									=			[
            f'''This is another {tokenizer.mask_token} test''',
        ]
        return fill_masker, examples
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       List[Any]									=			fill_masker.tokenizer
        lowercase_			:       List[Any]									=			fill_masker.model

        lowercase_			:       List[Any]									=			fill_masker(
            f'''This is a {tokenizer.mask_token}'''  ,)
        self.assertEqual(
            __UpperCamelCase  ,[
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
            ]  ,)

        lowercase_			:       List[str]									=			fill_masker([f'''This is a {tokenizer.mask_token}''']    )
        self.assertEqual(
            __UpperCamelCase  ,[
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
            ]  ,)

        lowercase_			:       Union[str, Any]									=			fill_masker([f'''This is a {tokenizer.mask_token}''', f'''Another {tokenizer.mask_token} great test.''']    )
        self.assertEqual(
            __UpperCamelCase  ,[
                [
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                ],
                [
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                ],
            ]  ,)

        with self.assertRaises(__UpperCamelCase    ):
             fill_masker([None]    )
        # No mask_token is not supported
        with self.assertRaises(__UpperCamelCase    ):
             fill_masker('This is'    )

        self.run_test_top_k(__UpperCamelCase  ,__UpperCamelCase    )
        self.run_test_targets(__UpperCamelCase  ,__UpperCamelCase    )
        self.run_test_top_k_targets(__UpperCamelCase  ,__UpperCamelCase    )
        self.fill_mask_with_duplicate_targets_and_top_k(__UpperCamelCase  ,__UpperCamelCase    )
        self.fill_mask_with_multiple_masks(__UpperCamelCase  ,__UpperCamelCase    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Any:


        '''simple docstring'''


        lowercase_			:       int									=			tokenizer.get_vocab()
        lowercase_			:       Optional[Any]									=			sorted(vocab.keys()    )[:2]
        # Pipeline argument
        lowercase_			:       int									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase  ,targets=__UpperCamelCase    )
        lowercase_			:       Dict									=			fill_masker(f'''This is a {tokenizer.mask_token}'''    )
        self.assertEqual(
            __UpperCamelCase  ,[
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
            ]  ,)
        lowercase_			:       Dict									=			{vocab[el] for el in targets}
        self.assertEqual({el['token'] for el in outputs}  ,__UpperCamelCase    )
        lowercase_			:       List[str]									=			[tokenizer.decode([x]    ) for x in target_ids]
        self.assertEqual({el['token_str'] for el in outputs}  ,set(__UpperCamelCase    )    )

        # Call argument
        lowercase_			:       str									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase    )
        lowercase_			:       Dict									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,targets=__UpperCamelCase    )
        self.assertEqual(
            __UpperCamelCase  ,[
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
            ]  ,)
        lowercase_			:       List[str]									=			{vocab[el] for el in targets}
        self.assertEqual({el['token'] for el in outputs}  ,__UpperCamelCase    )
        lowercase_			:       List[Any]									=			[tokenizer.decode([x]    ) for x in target_ids]
        self.assertEqual({el['token_str'] for el in outputs}  ,set(__UpperCamelCase    )    )

        # Score equivalence
        lowercase_			:       List[Any]									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,targets=__UpperCamelCase    )
        lowercase_			:       Optional[Any]									=			[top_mask['token_str'] for top_mask in outputs]
        lowercase_			:       int									=			[top_mask['score'] for top_mask in outputs]

        # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
        if set(__UpperCamelCase    ) == set(__UpperCamelCase    ):
             lowercase_			:       Union[str, Any]									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,targets=__UpperCamelCase    )
             lowercase_			:       List[str]									=			[top_mask['score'] for top_mask in unmasked_targets]
             self.assertEqual(nested_simplify(__UpperCamelCase    )  ,nested_simplify(__UpperCamelCase    )    )

        # Raises with invalid
        with self.assertRaises(__UpperCamelCase    ):
             lowercase_			:       Any									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,targets=[]    )
        # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
        if "" not in tokenizer.get_vocab():
             with self.assertRaises(__UpperCamelCase    ):
                  lowercase_			:       str									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,targets=['']    )
             with self.assertRaises(__UpperCamelCase    ):
                  lowercase_			:       Any									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,targets=''    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       Optional[int]									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase  ,top_k=2    )
        lowercase_			:       Dict									=			fill_masker(f'''This is a {tokenizer.mask_token}'''    )
        self.assertEqual(
            __UpperCamelCase  ,[
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
            ]  ,)

        lowercase_			:       str									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase    )
        lowercase_			:       Dict									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,top_k=2    )
        self.assertEqual(
            __UpperCamelCase  ,[
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
            ]  ,)
        self.assertEqual(nested_simplify(__UpperCamelCase    )  ,nested_simplify(__UpperCamelCase    )    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Dict:


        '''simple docstring'''


        lowercase_			:       Dict									=			tokenizer.get_vocab()
        lowercase_			:       List[str]									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase    )

        # top_k=2, ntargets=3
        lowercase_			:       Optional[Any]									=			sorted(vocab.keys()    )[:3]
        lowercase_			:       Tuple									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,top_k=2  ,targets=__UpperCamelCase    )

        # If we use the most probably targets, and filter differently, we should still
        # have the same results
        lowercase_			:       List[Any]									=			[el['token_str'] for el in sorted(__UpperCamelCase  ,key=lambda __UpperCamelCase    : x["score"]  ,reverse=__UpperCamelCase    )]
        # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
        if set(__UpperCamelCase    ).issubset(__UpperCamelCase    ):
             lowercase_			:       List[str]									=			fill_masker(f'''This is a {tokenizer.mask_token}'''  ,top_k=3  ,targets=__UpperCamelCase    )
             # They should yield exactly the same result
             self.assertEqual(nested_simplify(__UpperCamelCase    )  ,nested_simplify(__UpperCamelCase    )    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Any:


        '''simple docstring'''


        lowercase_			:       Optional[int]									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase    )
        lowercase_			:       Optional[Any]									=			tokenizer.get_vocab()
        # String duplicates + id duplicates
        lowercase_			:       Dict									=			sorted(vocab.keys()    )[:3]
        lowercase_			:       List[str]									=			[targets[0], targets[1], targets[0], targets[2], targets[1]]
        lowercase_			:       str									=			fill_masker(f'''My name is {tokenizer.mask_token}'''  ,targets=__UpperCamelCase  ,top_k=10    )

        # The target list contains duplicates, so we can't output more
        # than them
        self.assertEqual(len(__UpperCamelCase    )  ,3    )







   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase    )			->   str:


        '''simple docstring'''


        lowercase_			:       Union[str, Any]									=			FillMaskPipeline(model=__UpperCamelCase  ,tokenizer=__UpperCamelCase    )

        lowercase_			:       str									=			fill_masker(
            f'''This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'''  ,top_k=2    )
        self.assertEqual(
            __UpperCamelCase  ,[
                [
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                ],
                [
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                ],
                [
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                    {'sequence': ANY(__UpperCamelCase    ), 'score': ANY(__UpperCamelCase    ), 'token': ANY(__UpperCamelCase    ), 'token_str': ANY(__UpperCamelCase    )},
                ],
            ]  ,)


"""simple docstring"""







# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

__SCREAMING_SNAKE_CASE								="2.13.1"

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse("3.7"):
       raise ImportWarning(
           "To use `datasets`, Python>=3.7 is required, and the current version of Python doesn't match this condition."
       )

if version.parse(pyarrow.__version__).major < 8:
       raise ImportWarning(
           "To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn't match this condition.\n"
           "If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`."
       )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

__SCREAMING_SNAKE_CASE								=concatenate_datasets
__SCREAMING_SNAKE_CASE								=DownloadConfig
__SCREAMING_SNAKE_CASE								=DownloadManager
__SCREAMING_SNAKE_CASE								=DownloadMode
__SCREAMING_SNAKE_CASE								=DownloadConfig
__SCREAMING_SNAKE_CASE								=DownloadMode
__SCREAMING_SNAKE_CASE								=DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager

