"""simple docstring"""







from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import numpy as np
       import tensorflow as tf

       from transformers import (
           TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
           FlaubertConfig,
           TFFlaubertForMultipleChoice,
           TFFlaubertForQuestionAnsweringSimple,
           TFFlaubertForSequenceClassification,
           TFFlaubertForTokenClassification,
           TFFlaubertModel,
           TFFlaubertWithLMHeadModel,
       )







class  UpperCamelCase	:
   def __init__(    self  ,__UpperCamelCase  ,)			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       Optional[Any]									=			parent
        lowercase_			:       Optional[int]									=			13
        lowercase_			:       List[Any]									=			7
        lowercase_			:       str									=			True
        lowercase_			:       Any									=			True
        lowercase_			:       str									=			True
        lowercase_			:       Any									=			True
        lowercase_			:       str									=			True
        lowercase_			:       List[str]									=			False
        lowercase_			:       Optional[int]									=			False
        lowercase_			:       Dict									=			False
        lowercase_			:       Dict									=			2
        lowercase_			:       Optional[Any]									=			99
        lowercase_			:       Optional[Any]									=			0
        lowercase_			:       str									=			32
        lowercase_			:       int									=			2
        lowercase_			:       List[str]									=			4
        lowercase_			:       List[str]									=			0.1
        lowercase_			:       Optional[int]									=			0.1
        lowercase_			:       Dict									=			512
        lowercase_			:       str									=			16
        lowercase_			:       Optional[Any]									=			2
        lowercase_			:       Union[str, Any]									=			0.02
        lowercase_			:       int									=			3
        lowercase_			:       List[str]									=			4
        lowercase_			:       List[str]									=			'last'
        lowercase_			:       Optional[int]									=			True
        lowercase_			:       Optional[Any]									=			None
        lowercase_			:       str									=			0
   def     _UpperCAmelCase     (    self    )			->   Any:


        '''simple docstring'''


        lowercase_			:       List[Any]									=			ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size    )
        lowercase_			:       int									=			random_attention_mask([self.batch_size, self.seq_length]  ,dtype=tf.floataa    )

        lowercase_			:       Union[str, Any]									=			None
        if self.use_input_lengths:
             lowercase_			:       Dict									=			(
                 ids_tensor([self.batch_size]  ,vocab_size=2    ) + self.seq_length - 2
             )  # small variation of seq_length

        lowercase_			:       Dict									=			None
        if self.use_token_type_ids:
             lowercase_			:       List[Any]									=			ids_tensor([self.batch_size, self.seq_length]  ,self.n_langs    )

        lowercase_			:       Optional[Any]									=			None
        lowercase_			:       Optional[Any]									=			None
        lowercase_			:       Union[str, Any]									=			None
        if self.use_labels:
             lowercase_			:       int									=			ids_tensor([self.batch_size]  ,self.type_sequence_label_size    )
             lowercase_			:       Optional[int]									=			ids_tensor([self.batch_size, self.seq_length]  ,self.num_labels    )
             lowercase_			:       Dict									=			ids_tensor([self.batch_size]  ,2  ,dtype=tf.floataa    )
             lowercase_			:       Dict									=			ids_tensor([self.batch_size]  ,self.num_choices    )

        lowercase_			:       Optional[int]									=			FlaubertConfig(
            vocab_size=self.vocab_size  ,n_special=self.n_special  ,emb_dim=self.hidden_size  ,n_layers=self.num_hidden_layers  ,n_heads=self.num_attention_heads  ,dropout=self.hidden_dropout_prob  ,attention_dropout=self.attention_probs_dropout_prob  ,gelu_activation=self.gelu_activation  ,sinusoidal_embeddings=self.sinusoidal_embeddings  ,asm=self.asm  ,causal=self.causal  ,n_langs=self.n_langs  ,max_position_embeddings=self.max_position_embeddings  ,initializer_range=self.initializer_range  ,summary_type=self.summary_type  ,use_proj=self.use_proj  ,bos_token_id=self.bos_token_id  ,)

        return (
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
            choice_labels,
            input_mask,
        )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,)			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       List[Any]									=			TFFlaubertModel(config=__UpperCamelCase    )
        lowercase_			:       Any									=			{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
        lowercase_			:       int									=			model(__UpperCamelCase    )

        lowercase_			:       List[str]									=			[input_ids, input_mask]
        lowercase_			:       Tuple									=			model(__UpperCamelCase    )
        self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,)			->   List[Any]:


        '''simple docstring'''


        lowercase_			:       List[Any]									=			TFFlaubertWithLMHeadModel(__UpperCamelCase    )

        lowercase_			:       Tuple									=			{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
        lowercase_			:       Optional[Any]									=			model(__UpperCamelCase    )

        self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.vocab_size)    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,)			->   Any:


        '''simple docstring'''


        lowercase_			:       List[str]									=			TFFlaubertForQuestionAnsweringSimple(__UpperCamelCase    )

        lowercase_			:       List[str]									=			{'input_ids': input_ids, 'lengths': input_lengths}

        lowercase_			:       Tuple									=			model(__UpperCamelCase    )

        self.parent.assertEqual(result.start_logits.shape  ,(self.batch_size, self.seq_length)    )
        self.parent.assertEqual(result.end_logits.shape  ,(self.batch_size, self.seq_length)    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,)			->   Tuple:


        '''simple docstring'''


        lowercase_			:       Dict									=			TFFlaubertForSequenceClassification(__UpperCamelCase    )

        lowercase_			:       Any									=			{'input_ids': input_ids, 'lengths': input_lengths}

        lowercase_			:       Any									=			model(__UpperCamelCase    )

        self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.type_sequence_label_size)    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,)			->   Dict:


        '''simple docstring'''


        lowercase_			:       Any									=			self.num_labels
        lowercase_			:       Tuple									=			TFFlaubertForTokenClassification(config=__UpperCamelCase    )
        lowercase_			:       int									=			{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
        lowercase_			:       str									=			model(__UpperCamelCase    )
        self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.num_labels)    )
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,)			->   List[Any]:


        '''simple docstring'''


        lowercase_			:       Any									=			self.num_choices
        lowercase_			:       Any									=			TFFlaubertForMultipleChoice(config=__UpperCamelCase    )
        lowercase_			:       Optional[Any]									=			tf.tile(tf.expand_dims(__UpperCamelCase  ,1    )  ,(1, self.num_choices, 1)    )
        lowercase_			:       Dict									=			tf.tile(tf.expand_dims(__UpperCamelCase  ,1    )  ,(1, self.num_choices, 1)    )
        lowercase_			:       Optional[Any]									=			tf.tile(tf.expand_dims(__UpperCamelCase  ,1    )  ,(1, self.num_choices, 1)    )
        lowercase_			:       List[str]									=			{
            'input_ids': multiple_choice_inputs_ids,
            'attention_mask': multiple_choice_input_mask,
            'token_type_ids': multiple_choice_token_type_ids,
        }
        lowercase_			:       Tuple									=			model(__UpperCamelCase    )
        self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_choices)    )







   def     _UpperCAmelCase     (    self    )			->   int:


        '''simple docstring'''


        lowercase_			:       Optional[Any]									=			self.prepare_config_and_inputs()
        (
            (
            lowercase_
        )							,						(
            lowercase_
        )							,						(
            lowercase_
        )							,						(
            lowercase_
        )							,						(
            lowercase_
        )							,						(
            lowercase_
        )							,						(
            lowercase_
        )							,						(
            lowercase_
        )							,						(
            lowercase_
        )							,						
        )			:       Optional[int]									=			config_and_inputs
        lowercase_			:       List[str]									=			{
            'input_ids': input_ids,
            'token_type_ids': token_type_ids,
            'langs': token_type_ids,
            'lengths': input_lengths,
        }
        return config, inputs_dict







@require_tf
class  UpperCamelCase	(    lowercase_		,   lowercase_		,   unittest.TestCase						):
   lowercase							=						(
       (
           TFFlaubertModel,
           TFFlaubertWithLMHeadModel,
           TFFlaubertForSequenceClassification,
           TFFlaubertForQuestionAnsweringSimple,
           TFFlaubertForTokenClassification,
           TFFlaubertForMultipleChoice,
       )
       if is_tf_available()
       else ()
   )
   lowercase							=						(
       (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
   )  # TODO (PVP): Check other models whether language generation is also applicable
   lowercase							=						(
       {
           'feature-extraction': TFFlaubertModel,
           'fill-mask': TFFlaubertWithLMHeadModel,
           'question-answering': TFFlaubertForQuestionAnsweringSimple,
           'text-classification': TFFlaubertForSequenceClassification,
           'token-classification': TFFlaubertForTokenClassification,
           'zero-shot': TFFlaubertForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   lowercase							=						False
   lowercase							=						False
   def     _UpperCAmelCase     (    self  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase  ,__UpperCamelCase    )			->   Dict:


        '''simple docstring'''


        if (
            pipeline_test_casse_name == "QAPipelineTests"
            and tokenizer_name is not None
            and not tokenizer_name.endswith('Fast'    )
        ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

        return False
   def     _UpperCAmelCase     (    self    )			->   Union[str, Any]:


        '''simple docstring'''


        lowercase_			:       int									=			TFFlaubertModelTester(self    )
        lowercase_			:       Dict									=			ConfigTester(self  ,config_class=__UpperCamelCase  ,emb_dim=37    )
   def     _UpperCAmelCase     (    self    )			->   Dict:


        '''simple docstring'''


        self.config_tester.run_common_tests()
   def     _UpperCAmelCase     (    self    )			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       Any									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_model(*__UpperCamelCase    )
   def     _UpperCAmelCase     (    self    )			->   int:


        '''simple docstring'''


        lowercase_			:       Dict									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_lm_head(*__UpperCamelCase    )
   def     _UpperCAmelCase     (    self    )			->   List[str]:


        '''simple docstring'''


        lowercase_			:       List[str]									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_qa(*__UpperCamelCase    )
   def     _UpperCAmelCase     (    self    )			->   Tuple:


        '''simple docstring'''


        lowercase_			:       Dict									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_sequence_classif(*__UpperCamelCase    )
   def     _UpperCAmelCase     (    self    )			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       Dict									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_for_token_classification(*__UpperCamelCase    )
   def     _UpperCAmelCase     (    self    )			->   Tuple:


        '''simple docstring'''


        lowercase_			:       Optional[Any]									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_for_multiple_choice(*__UpperCamelCase    )







   @slow
   def     _UpperCAmelCase     (    self    )			->   Union[str, Any]:


        '''simple docstring'''


        for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowercase_			:       List[str]									=			TFFlaubertModel.from_pretrained(__UpperCamelCase    )
             self.assertIsNotNone(__UpperCamelCase    )







@require_tf
@require_sentencepiece
@require_tokenizers
class  UpperCamelCase	(    unittest.TestCase						):
   @slow
   def     _UpperCAmelCase     (    self    )			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       List[str]									=			TFFlaubertModel.from_pretrained('jplu/tf-flaubert-small-cased'    )

        lowercase_			:       Dict									=			tf.convert_to_tensor(
            [[0, 158, 735, 2592, 1424, 6727, 82, 1]]  ,dtype=tf.intaa  ,)  # "J'aime flaubert !"

        lowercase_			:       Dict									=			model(__UpperCamelCase    )[0]
        lowercase_			:       List[str]									=			tf.TensorShape((1, 8, 512)    )
        self.assertEqual(output.shape  ,__UpperCamelCase    )
        # compare the actual values for a slice.
        lowercase_			:       Union[str, Any]									=			tf.convert_to_tensor(
            [
                [
                    [-1.876_8773, -1.56_6555, 0.2707_2418],
                    [-1.692_0038, -0.587_3505, 1.932_9599],
                    [-2.956_3985, -1.699_3835, 1.797_2052],
                ]
            ]  ,dtype=tf.floataa  ,)

        self.assertTrue(np.allclose(output[:, :3, :3].numpy()  ,expected_slice.numpy()  ,atol=1e-4    )    )


"""simple docstring"""







# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

__SCREAMING_SNAKE_CASE								="2.13.1"

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse("3.7"):
       raise ImportWarning(
           "To use `datasets`, Python>=3.7 is required, and the current version of Python doesn't match this condition."
       )

if version.parse(pyarrow.__version__).major < 8:
       raise ImportWarning(
           "To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn't match this condition.\n"
           "If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`."
       )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

__SCREAMING_SNAKE_CASE								=concatenate_datasets
__SCREAMING_SNAKE_CASE								=DownloadConfig
__SCREAMING_SNAKE_CASE								=DownloadManager
__SCREAMING_SNAKE_CASE								=DownloadMode
__SCREAMING_SNAKE_CASE								=DownloadConfig
__SCREAMING_SNAKE_CASE								=DownloadMode
__SCREAMING_SNAKE_CASE								=DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager

