
'''simple docstring'''







from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch


@require_torch
@require_tf
class 			lowerCAmelCase (				__lowerCamelCase			):




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= SMALL_MODEL_IDENTIFIER
								_SCREAMING_SNAKE_CASE				= 'pt'
								_SCREAMING_SNAKE_CASE				= 'tf'




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= AutoModel.from_pretrained(self.test_model      )
								model_pt.save_pretrained(UpperCAmelCase_      )




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= TFAutoModel.from_pretrained(self.test_model					,						from_pt=UpperCAmelCase_      )
								model_tf.save_pretrained(UpperCAmelCase_      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= 'mock_framework'

								# Framework provided - return whatever the user provides
								_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(self.test_model					,						UpperCAmelCase_      )
								self.assertEqual(UpperCAmelCase_					,						UpperCAmelCase_      )

								# Local checkpoint and framework provided - return provided framework
								# PyTorch checkpoint
								with TemporaryDirectory() as local_pt_ckpt:
														self._setup_pt_ckpt(UpperCAmelCase_      )
														_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(UpperCAmelCase_					,						UpperCAmelCase_      )
														self.assertEqual(UpperCAmelCase_					,						UpperCAmelCase_      )

								# TensorFlow checkpoint
								with TemporaryDirectory() as local_tf_ckpt:
														self._setup_tf_ckpt(UpperCAmelCase_      )
														_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(UpperCAmelCase_					,						UpperCAmelCase_      )
														self.assertEqual(UpperCAmelCase_					,						UpperCAmelCase_      )




		def     lowercase       (				self      ):
								# PyTorch checkpoint
								with TemporaryDirectory() as local_pt_ckpt:
														self._setup_pt_ckpt(UpperCAmelCase_      )
														_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(UpperCAmelCase_      )
														self.assertEqual(UpperCAmelCase_					,						self.framework_pt      )

								# TensorFlow checkpoint
								with TemporaryDirectory() as local_tf_ckpt:
														self._setup_tf_ckpt(UpperCAmelCase_      )
														_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(UpperCAmelCase_      )
														self.assertEqual(UpperCAmelCase_					,						self.framework_tf      )

								# Invalid local checkpoint
								with TemporaryDirectory() as local_invalid_ckpt:
														with self.assertRaises(UpperCAmelCase_      ):
																				_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(UpperCAmelCase_      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= MagicMock(return_value=UpperCAmelCase_      )
								with patch("transformers.onnx.features.is_tf_available"					,						UpperCAmelCase_      ):
														_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(self.test_model      )
														self.assertEqual(UpperCAmelCase_					,						self.framework_pt      )

								# PyTorch not in environment -> use TensorFlow
								_SCREAMING_SNAKE_CASE				= MagicMock(return_value=UpperCAmelCase_      )
								with patch("transformers.onnx.features.is_torch_available"					,						UpperCAmelCase_      ):
														_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(self.test_model      )
														self.assertEqual(UpperCAmelCase_					,						self.framework_tf      )

								# Both in environment -> use PyTorch
								_SCREAMING_SNAKE_CASE				= MagicMock(return_value=UpperCAmelCase_      )
								_SCREAMING_SNAKE_CASE				= MagicMock(return_value=UpperCAmelCase_      )
								with patch("transformers.onnx.features.is_tf_available"					,						UpperCAmelCase_      ), patch(
								    "transformers.onnx.features.is_torch_available"					,						UpperCAmelCase_      ):
														_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(self.test_model      )
														self.assertEqual(UpperCAmelCase_					,						self.framework_pt      )

								# Both not in environment -> raise error
								_SCREAMING_SNAKE_CASE				= MagicMock(return_value=UpperCAmelCase_      )
								_SCREAMING_SNAKE_CASE				= MagicMock(return_value=UpperCAmelCase_      )
								with patch("transformers.onnx.features.is_tf_available"					,						UpperCAmelCase_      ), patch(
								    "transformers.onnx.features.is_torch_available"					,						UpperCAmelCase_      ):
														with self.assertRaises(UpperCAmelCase_      ):
																				_SCREAMING_SNAKE_CASE				= FeaturesManager.determine_framework(self.test_model      )

'''simple docstring'''







from transformers import DistilBertTokenizer, DistilBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest


@require_tokenizers
class 			lowerCAmelCase (				__UpperCAmelCase			):
		a							:					Dict       =					DistilBertTokenizer
		a							:					Tuple       =					DistilBertTokenizerFast
		a							:					List[str]       =					True




		@slow
		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= DistilBertTokenizer.from_pretrained("distilbert-base-uncased"      )

								_SCREAMING_SNAKE_CASE				= tokenizer.encode("sequence builders"					,						add_special_tokens=UpperCamelCase      )
								_SCREAMING_SNAKE_CASE				= tokenizer.encode("multi-sequence build"					,						add_special_tokens=UpperCamelCase      )

								_SCREAMING_SNAKE_CASE				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase      )
								_SCREAMING_SNAKE_CASE				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase					,						UpperCamelCase      )

								assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
								assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
								    tokenizer.sep_token_id
								]