




"""simple docstring"""





import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
			import tensorflow as tf

if is_tensorflow_text_available():
			from transformers.models.bert import TFBertTokenizer


A   =       ["""bert-base-uncased""", """bert-base-cased"""]
A   =       """hf-internal-testing/tiny-bert-tf-only"""

if is_tf_available():



			class      a__				(	tf.keras.Model   ):


						def __init__(      self      :    Optional[Any]      ,  UpperCamelCase_      :    Optional[int]):


									"""simple docstring"""




									super().__init__()
									__UpperCAmelCase	: Union[str, Any]		=					tokenizer
									__UpperCAmelCase	: List[Any]		=					AutoConfig.from_pretrained(UpperCamelCase_)
									__UpperCAmelCase	: Tuple		=					TFAutoModel.from_config(UpperCamelCase_)


						def       a_     (      self      :    int      ,  UpperCamelCase_      :    str):


									"""simple docstring"""




									__UpperCAmelCase	: Tuple		=					self.tokenizer(UpperCamelCase_)
									__UpperCAmelCase	: Optional[int]		=					self.bert(**UpperCamelCase_)
									return out["pooler_output"]



@require_tf
@require_tensorflow_text
class      a__				(	unittest.TestCase   ):


			def       a_     (      self      :    List[str]):


						"""simple docstring"""




						super().setUp()

						__UpperCAmelCase	: str		=					[
						    BertTokenizer.from_pretrained(UpperCamelCase_) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
						]  # repeat for when fast_bert_tokenizer=false
						__UpperCAmelCase	: List[Any]		=					[TFBertTokenizer.from_pretrained(UpperCamelCase_) for checkpoint in TOKENIZER_CHECKPOINTS] + [
						    TFBertTokenizer.from_pretrained(UpperCamelCase_      ,  use_fast_bert_tokenizer=UpperCamelCase_)
						    for checkpoint in TOKENIZER_CHECKPOINTS
						]
						assert len(self.tokenizers) == len(self.tf_tokenizers)

						__UpperCAmelCase	: Tuple		=					[
						    "This is a straightforward English test sentence.",
						    "This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.",
						    "Now we're going to add some Chinese: 一 二 三 一二三",
						    "And some much more rare Chinese: 齉 堃 齉堃",
						    "Je vais aussi écrire en français pour tester les accents",
						    "Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ",
						]
						__UpperCAmelCase	: Union[str, Any]		=					list(zip(self.test_sentences      ,  self.test_sentences[::-1]))


			def       a_     (      self      :    List[str]):


						"""simple docstring"""




						for tokenizer, tf_tokenizer in zip(self.tokenizers      ,  self.tf_tokenizers):
									for test_inputs in (self.test_sentences, self.paired_sentences):
												__UpperCAmelCase	: Dict		=					tokenizer(UpperCamelCase_      ,  return_tensors="tf"      ,  padding="longest")
												__UpperCAmelCase	: List[Any]		=					tf_tokenizer(UpperCamelCase_)

												for key in python_outputs.keys():
															self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape))
															self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]      ,  tf.intaa) == tf_outputs[key]))


			@slow
			def       a_     (      self      :    Tuple):


						"""simple docstring"""




						for tf_tokenizer in self.tf_tokenizers:
									__UpperCAmelCase	: Optional[int]		=					tf_tokenizer(self.paired_sentences)
									__UpperCAmelCase	: int		=					tf_tokenizer(
									    text=[sentence[0] for sentence in self.paired_sentences]      ,  text_pair=[sentence[1] for sentence in self.paired_sentences]      ,  )
									for key in merged_outputs.keys():
												self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]      ,  tf.intaa) == separated_outputs[key]))


			@slow
			def       a_     (      self      :    Dict):


						"""simple docstring"""




						for tf_tokenizer in self.tf_tokenizers:
									__UpperCAmelCase	: Tuple		=					tf.function(UpperCamelCase_)
									for test_inputs in (self.test_sentences, self.paired_sentences):
												__UpperCAmelCase	: Tuple		=					tf.constant(UpperCamelCase_)
												__UpperCAmelCase	: Union[str, Any]		=					compiled_tokenizer(UpperCamelCase_)
												__UpperCAmelCase	: str		=					tf_tokenizer(UpperCamelCase_)

												for key in eager_outputs.keys():
															self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]))


			@slow
			def       a_     (      self      :    int):


						"""simple docstring"""




						for tf_tokenizer in self.tf_tokenizers:
									__UpperCAmelCase	: Optional[Any]		=					ModelToSave(tokenizer=UpperCamelCase_)
									__UpperCAmelCase	: Tuple		=					tf.convert_to_tensor(self.test_sentences)
									__UpperCAmelCase	: Tuple		=					model(UpperCamelCase_)  # Build model with some sample inputs
									with TemporaryDirectory() as tempdir:
												__UpperCAmelCase	: Tuple		=					Path(UpperCamelCase_) / "saved.model"
												model.save(UpperCamelCase_)
												__UpperCAmelCase	: Optional[int]		=					tf.keras.models.load_model(UpperCamelCase_)
									__UpperCAmelCase	: Union[str, Any]		=					loaded_model(UpperCamelCase_)
									# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
									self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output))      ,  1e-5)








"""simple docstring"""





from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader



class      a__				(	__magic_name__   ):


			def __init__(      self      :    List[Any]      ,  UpperCamelCase_      :    pyspark.sql.DataFrame      ,  UpperCamelCase_      :    Optional[NamedSplit] = None      ,  UpperCamelCase_      :    Optional[Features] = None      ,  UpperCamelCase_      :    bool = True      ,  UpperCamelCase_      :    str = None      ,  UpperCamelCase_      :    bool = False      ,  UpperCamelCase_      :    str = None      ,  UpperCamelCase_      :    bool = True      ,  UpperCamelCase_      :    str = "arrow"      ,  **UpperCamelCase_      :    Tuple      ,  ):


						"""simple docstring"""




						super().__init__(
						    split=UpperCamelCase_      ,  features=UpperCamelCase_      ,  cache_dir=UpperCamelCase_      ,  keep_in_memory=UpperCamelCase_      ,  streaming=UpperCamelCase_      ,  **UpperCamelCase_      ,  )
						__UpperCAmelCase	: List[Any]		=					load_from_cache_file
						__UpperCAmelCase	: List[Any]		=					file_format
						__UpperCAmelCase	: str		=					Spark(
						    df=UpperCamelCase_      ,  features=UpperCamelCase_      ,  cache_dir=UpperCamelCase_      ,  working_dir=UpperCamelCase_      ,  **UpperCamelCase_      ,  )


			def       a_     (      self      :    Union[str, Any]):


						"""simple docstring"""




						if self.streaming:
									return self.builder.as_streaming_dataset(split=self.split)
						__UpperCAmelCase	: List[Any]		=					None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
						self.builder.download_and_prepare(
						    download_mode=UpperCamelCase_      ,  file_format=self._file_format      ,  )
						return self.builder.as_dataset(split=self.split)


