
"""simple docstring"""



import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							List[Any]							=					"Wav2Vec2FeatureExtractor"
				SCREAMING_SNAKE_CASE_:							List[Any]							=					"AutoTokenizer"


				def __init__(					self	:			Optional[int]				,     UpperCAmelCase_	:			Optional[Any]				,     UpperCAmelCase_	:			Dict  )		->  List[str]:







											"""simple docstring"""




											super().__init__(UpperCAmelCase_				,     UpperCAmelCase_  )
											_lowerCAmelCase  	=			self.feature_extractor
											_lowerCAmelCase  	=			False


				@classmethod
				def 				__lowerCamelCase	(					cls	:			List[str]				,     UpperCAmelCase_	:			int				,     **UpperCAmelCase_	:			Dict  )		->  str:







											"""simple docstring"""




											try:
																		return super().from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
											except OSError:
																		warnings.warn(
																		    F"""Loading a tokenizer inside {cls.__name__} from a config that does not"""
																		    ' include a `tokenizer_class` attribute is deprecated and will be '
																		    'removed in v5. Please add `\'tokenizer_class\': \'Wav2Vec2CTCTokenizer\'`'
																		    ' attribute to either your `config.json` or `tokenizer_config.json` '
																		    'file to suppress this warning: '				,     UpperCAmelCase_				,     )

																		_lowerCAmelCase  	=			WavaVecaFeatureExtractor.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
																		_lowerCAmelCase  	=			WavaVecaCTCTokenizer.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )

																		return cls(feature_extractor=UpperCAmelCase_				,     tokenizer=UpperCAmelCase_  )


				def __call__(					self	:			Optional[int]				,     *UpperCAmelCase_	:			int				,     **UpperCAmelCase_	:			Optional[int]  )		->  List[str]:







											"""simple docstring"""




											if self._in_target_context_manager:
																		return self.current_processor(*UpperCAmelCase_				,     **UpperCAmelCase_  )

											if "raw_speech" in kwargs:
																		warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'  )
																		_lowerCAmelCase  	=			kwargs.pop('raw_speech'  )
											else:
																		_lowerCAmelCase  	=			kwargs.pop('audio'				,     UpperCAmelCase_  )
											_lowerCAmelCase  	=			kwargs.pop('sampling_rate'				,     UpperCAmelCase_  )
											_lowerCAmelCase  	=			kwargs.pop('text'				,     UpperCAmelCase_  )
											if len(UpperCAmelCase_  ) > 0:
																		_lowerCAmelCase  	=			args[0]
																		_lowerCAmelCase  	=			args[1:]

											if audio is None and text is None:
																		raise ValueError('You need to specify either an `audio` or `text` input to process.'  )

											if audio is not None:
																		_lowerCAmelCase  	=			self.feature_extractor(UpperCAmelCase_				,     *UpperCAmelCase_				,     sampling_rate=UpperCAmelCase_				,     **UpperCAmelCase_  )
											if text is not None:
																		_lowerCAmelCase  	=			self.tokenizer(UpperCAmelCase_				,     **UpperCAmelCase_  )

											if text is None:
																		return inputs
											elif audio is None:
																		return encodings
											else:
																		_lowerCAmelCase  	=			encodings['input_ids']
																		return inputs


				def 				__lowerCamelCase	(					self	:			Optional[int]				,     *UpperCAmelCase_	:			Tuple				,     **UpperCAmelCase_	:			int  )		->  Any:







											"""simple docstring"""




											if self._in_target_context_manager:
																		return self.current_processor.pad(*UpperCAmelCase_				,     **UpperCAmelCase_  )

											_lowerCAmelCase  	=			kwargs.pop('input_features'				,     UpperCAmelCase_  )
											_lowerCAmelCase  	=			kwargs.pop('labels'				,     UpperCAmelCase_  )
											if len(UpperCAmelCase_  ) > 0:
																		_lowerCAmelCase  	=			args[0]
																		_lowerCAmelCase  	=			args[1:]

											if input_features is not None:
																		_lowerCAmelCase  	=			self.feature_extractor.pad(UpperCAmelCase_				,     *UpperCAmelCase_				,     **UpperCAmelCase_  )
											if labels is not None:
																		_lowerCAmelCase  	=			self.tokenizer.pad(UpperCAmelCase_				,     **UpperCAmelCase_  )

											if labels is None:
																		return input_features
											elif input_features is None:
																		return labels
											else:
																		_lowerCAmelCase  	=			labels['input_ids']
																		return input_features


				def 				__lowerCamelCase	(					self	:			Optional[int]				,     *UpperCAmelCase_	:			List[str]				,     **UpperCAmelCase_	:			Optional[int]  )		->  str:







											"""simple docstring"""




											return self.tokenizer.batch_decode(*UpperCAmelCase_				,     **UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Dict				,     *UpperCAmelCase_	:			Any				,     **UpperCAmelCase_	:			Tuple  )		->  Optional[int]:







											"""simple docstring"""




											return self.tokenizer.decode(*UpperCAmelCase_				,     **UpperCAmelCase_  )







				@contextmanager
				def 				__lowerCamelCase	(					self	:			Dict  )		->  str:







											"""simple docstring"""




											warnings.warn(
											    '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
											    'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
											    'your audio inputs, or in a separate call.'  )
											_lowerCAmelCase  	=			True
											_lowerCAmelCase  	=			self.tokenizer
											yield
											_lowerCAmelCase  	=			self.feature_extractor
											_lowerCAmelCase  	=			False








"""simple docstring"""



import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					None
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							Any							=					True
				SCREAMING_SNAKE_CASE_:							List[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Optional[int]							=					"tokenizer_file"
				SCREAMING_SNAKE_CASE_:							str							=					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  str:







											"""simple docstring"""




											super().setUp()
											_lowerCAmelCase  	=			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'  )
											tokenizer.save_pretrained(self.tmpdirname  )


				def 				__lowerCamelCase	(					self	:			List[str]				,     **UpperCAmelCase_	:			Any  )		->  Optional[Any]:







											"""simple docstring"""




											kwargs.update(self.special_tokens_map  )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  str:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()

											_lowerCAmelCase  	=			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											_lowerCAmelCase  	=			[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

											_lowerCAmelCase  	=			tokenizer.batch_encode_plus(UpperCAmelCase_  )['input_ids']
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			tokenizer.batch_decode(UpperCAmelCase_  )
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any				,     UpperCAmelCase_	:			List[str]=6  )		->  Any:







											"""simple docstring"""




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																									_lowerCAmelCase  	=			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
																									# tokenizer_r.pad_token = None # Hotfixing padding = None
																									# Simple input
																									_lowerCAmelCase  	=			'This is a simple input'
																									_lowerCAmelCase  	=			['This is a simple input 1', 'This is a simple input 2']
																									_lowerCAmelCase  	=			('This is a simple input', 'This is a pair')
																									_lowerCAmelCase  	=			[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									try:
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )

																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																									except ValueError:
																																self.fail('Bloom Tokenizer should be able to deal with padding'  )

																									_lowerCAmelCase  	=			None  # Hotfixing padding = None
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )


				def 				__lowerCamelCase	(					self	:			str  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()
											_lowerCAmelCase  	=			load_dataset('xnli'				,     'all_languages'				,     split='test'				,     streaming=UpperCAmelCase_  )

											_lowerCAmelCase  	=			next(iter(UpperCAmelCase_  )  )['premise']  # pick up one data
											_lowerCAmelCase  	=			list(sample_data.values()  )

											_lowerCAmelCase  	=			list(map(tokenizer.encode				,     UpperCAmelCase_  )  )
											_lowerCAmelCase  	=			[tokenizer.decode(UpperCAmelCase_				,     clean_up_tokenization_spaces=UpperCAmelCase_  ) for x in output_tokens]
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )







				def 				__lowerCamelCase	(					self	:			Tuple  )		->  List[Any]:







											"""simple docstring"""




											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )				,     1  )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )				,     1  )






