


'''simple docstring'''

import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase				: str							    =				logging.get_logger(__name__)

_UpperCAmelCase				: Tuple							    =				{"""vocab_file""": """spiece.model"""}

_UpperCAmelCase				: str							    =				{
    """vocab_file""": {
        """google/bigbird-roberta-base""": """https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model""",
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"""
        ),
    }
}

_UpperCAmelCase				: Any							    =				{
    """google/bigbird-roberta-base""": 4_0_9_6,
    """google/bigbird-roberta-large""": 4_0_9_6,
    """google/bigbird-base-trivia-itc""": 4_0_9_6,
}


class       a__				( __A      ):







				"""simple docstring"""



				__UpperCamelCase    :		List[str]	=	VOCAB_FILES_NAMES
				__UpperCamelCase    :		Dict	=	PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase    :		str	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase    :		Union[str, Any]	=	['input_ids', 'attention_mask']
				__UpperCamelCase    :		List[int]	=	[]




				def __init__(self       ,			__lowercase       ,			__lowercase="<unk>"       ,			__lowercase="<s>"       ,			__lowercase="</s>"       ,			__lowercase="<pad>"       ,			__lowercase="[SEP]"       ,			__lowercase="[MASK]"       ,			__lowercase="[CLS]"       ,			__lowercase = None       ,			**__lowercase       ,			):
										__lowerCAmelCase							       =				AddedToken(__lowercase       ,			lstrip=__lowercase       ,			rstrip=__lowercase		) if isinstance(__lowercase       ,			__lowercase		) else bos_token
										__lowerCAmelCase							       =				AddedToken(__lowercase       ,			lstrip=__lowercase       ,			rstrip=__lowercase		) if isinstance(__lowercase       ,			__lowercase		) else eos_token
										__lowerCAmelCase							       =				AddedToken(__lowercase       ,			lstrip=__lowercase       ,			rstrip=__lowercase		) if isinstance(__lowercase       ,			__lowercase		) else unk_token
										__lowerCAmelCase							       =				AddedToken(__lowercase       ,			lstrip=__lowercase       ,			rstrip=__lowercase		) if isinstance(__lowercase       ,			__lowercase		) else pad_token
										__lowerCAmelCase							       =				AddedToken(__lowercase       ,			lstrip=__lowercase       ,			rstrip=__lowercase		) if isinstance(__lowercase       ,			__lowercase		) else cls_token
										__lowerCAmelCase							       =				AddedToken(__lowercase       ,			lstrip=__lowercase       ,			rstrip=__lowercase		) if isinstance(__lowercase       ,			__lowercase		) else sep_token

										# Mask token behave like a normal word, i.e. include the space before it
										__lowerCAmelCase							       =				AddedToken(__lowercase       ,			lstrip=__lowercase       ,			rstrip=__lowercase		) if isinstance(__lowercase       ,			__lowercase		) else mask_token

										__lowerCAmelCase							       =				{} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    bos_token=__lowercase       ,			eos_token=__lowercase       ,			unk_token=__lowercase       ,			pad_token=__lowercase       ,			sep_token=__lowercase       ,			mask_token=__lowercase       ,			cls_token=__lowercase       ,			sp_model_kwargs=self.sp_model_kwargs       ,			**__lowercase       ,			)

										__lowerCAmelCase							       =				vocab_file

										__lowerCAmelCase							       =				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
										self.sp_model.Load(__lowercase		)




				@property
				def 					_snake_case  (self		):
										return self.sp_model.get_piece_size()




				def 					_snake_case  (self		):
										__lowerCAmelCase							       =				{self.convert_ids_to_tokens(__lowercase		): i for i in range(self.vocab_size		)}
										vocab.update(self.added_tokens_encoder		)
										return vocab




				def __getstate__(self		):
										__lowerCAmelCase							       =				self.__dict__.copy()
										__lowerCAmelCase							       =				None
										return state




				def __setstate__(self       ,			__lowercase		):
										__lowerCAmelCase							       =				d

										# for backward compatibility
										if not hasattr(self       ,			'''sp_model_kwargs'''		):
																__lowerCAmelCase							       =				{}

										__lowerCAmelCase							       =				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
										self.sp_model.Load(self.vocab_file		)




				def 					_snake_case  (self       ,			__lowercase		):
										return self.sp_model.encode(__lowercase       ,			out_type=__lowercase		)




				def 					_snake_case  (self       ,			__lowercase		):
										return self.sp_model.piece_to_id(__lowercase		)




				def 					_snake_case  (self       ,			__lowercase		):
										__lowerCAmelCase							       =				self.sp_model.IdToPiece(__lowercase		)
										return token




				def 					_snake_case  (self       ,			__lowercase		):
										__lowerCAmelCase							       =				[]
										__lowerCAmelCase							       =				''''''
										__lowerCAmelCase							       =				False
										for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																						if not prev_is_special:
																												out_string += " "
																						out_string += self.sp_model.decode(__lowercase		) + token
																						__lowerCAmelCase							       =				True
																						__lowerCAmelCase							       =				[]
																else:
																						current_sub_tokens.append(__lowercase		)
																						__lowerCAmelCase							       =				False
										out_string += self.sp_model.decode(__lowercase		)
										return out_string.strip()




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase = False       ,			__lowercase = None       ,			__lowercase = True       ,			**__lowercase       ,			):
										__lowerCAmelCase							       =				kwargs.pop('''use_source_tokenizer'''       ,			__lowercase		)

										__lowerCAmelCase							       =				self.convert_ids_to_tokens(__lowercase       ,			skip_special_tokens=__lowercase		)

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										__lowerCAmelCase							       =				[]
										__lowerCAmelCase							       =				[]
										for token in filtered_tokens:
																if skip_special_tokens and token in self.all_special_ids:
																						continue
																if token in self.added_tokens_encoder:
																						if current_sub_text:
																												sub_texts.append(self.convert_tokens_to_string(__lowercase		)		)
																												__lowerCAmelCase							       =				[]
																						sub_texts.append(__lowercase		)
																else:
																						current_sub_text.append(__lowercase		)
										if current_sub_text:
																sub_texts.append(self.convert_tokens_to_string(__lowercase		)		)

										# Mimic the behavior of the Rust tokenizer:
										# No space before [MASK] and [SEP]
										if spaces_between_special_tokens:
																__lowerCAmelCase							       =				re.sub(R''' (\[(MASK|SEP)\])'''       ,			R'''\1'''       ,			''' '''.join(__lowercase		)		)
										else:
																__lowerCAmelCase							       =				''''''.join(__lowercase		)

										__lowerCAmelCase							       =				(
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
																__lowerCAmelCase							       =				self.clean_up_tokenization(__lowercase		)
																return clean_text
										else:
																return text




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase = None		):
										if not os.path.isdir(__lowercase		):
																logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
																return
										__lowerCAmelCase							       =				os.path.join(
										    __lowercase       ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

										if os.path.abspath(self.vocab_file		) != os.path.abspath(__lowercase		) and os.path.isfile(self.vocab_file		):
																copyfile(self.vocab_file       ,			__lowercase		)
										elif not os.path.isfile(self.vocab_file		):
																with open(__lowercase       ,			'''wb'''		) as fi:
																						__lowerCAmelCase							       =				self.sp_model.serialized_model_proto()
																						fi.write(__lowercase		)

										return (out_vocab_file,)




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase = None		):
										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										__lowerCAmelCase							       =				[self.cls_token_id]
										__lowerCAmelCase							       =				[self.sep_token_id]
										return cls + token_ids_a + sep + token_ids_a + sep




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase = None       ,			__lowercase = False		):
										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=__lowercase       ,			token_ids_a=__lowercase       ,			already_has_special_tokens=__lowercase		)

										if token_ids_a is None:
																return [1] + ([0] * len(__lowercase		)) + [1]
										return [1] + ([0] * len(__lowercase		)) + [1] + ([0] * len(__lowercase		)) + [1]




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase = None		):
										__lowerCAmelCase							       =				[self.sep_token_id]
										__lowerCAmelCase							       =				[self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep		) * [0]
										return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





'''simple docstring'''

import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


_UpperCAmelCase				: Tuple							    =				random.Random()

if is_torch_available():
	import torch

def 							__magic_name__(						lowerCamelCase,				lowerCamelCase=1.0,				lowerCamelCase=None,				lowerCamelCase=None):
						if rng is None:
												__lowerCAmelCase							       =				global_rng

						__lowerCAmelCase							       =				[]
						for batch_idx in range(shape[0]):
												values.append([])
												for _ in range(shape[1]):
																		values[-1].append(rng.random() * scale)

						return values


class       a__				( unittest.TestCase      ):







				"""simple docstring"""



				def __init__(self       ,			__lowercase       ,			__lowercase=7       ,			__lowercase=4_00       ,			__lowercase=20_00       ,			__lowercase=1       ,			__lowercase=0.0       ,			__lowercase=1_60_00       ,			__lowercase=True       ,			__lowercase=True       ,			):
										__lowerCAmelCase							       =				parent
										__lowerCAmelCase							       =				batch_size
										__lowerCAmelCase							       =				min_seq_length
										__lowerCAmelCase							       =				max_seq_length
										__lowerCAmelCase							       =				(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
										__lowerCAmelCase							       =				feature_size
										__lowerCAmelCase							       =				padding_value
										__lowerCAmelCase							       =				sampling_rate
										__lowerCAmelCase							       =				return_attention_mask
										__lowerCAmelCase							       =				do_normalize




				def 					_snake_case  (self		):
										return {
										    "feature_size": self.feature_size,
										    "padding_value": self.padding_value,
										    "sampling_rate": self.sampling_rate,
										    "return_attention_mask": self.return_attention_mask,
										    "do_normalize": self.do_normalize,
										}




				def 					_snake_case  (self       ,			__lowercase=False       ,			__lowercase=False		):
										def _flatten(__lowercase		):
																return list(itertools.chain(*__lowercase		)		)

										if equal_length:
																__lowerCAmelCase							       =				floats_list((self.batch_size, self.max_seq_length)		)
										else:
																# make sure that inputs increase in size
																__lowerCAmelCase							       =				[
																    _flatten(floats_list((x, self.feature_size)		)		)
																    for x in range(self.min_seq_length       ,			self.max_seq_length       ,			self.seq_length_diff		)
																]

										if numpify:
																__lowerCAmelCase							       =				[np.asarray(__lowercase		) for x in speech_inputs]

										return speech_inputs


@require_torch
@require_torchaudio
class       a__				( __A							,						unittest.TestCase      ):







				"""simple docstring"""



				__UpperCamelCase    :		Union[str, Any]	=	ASTFeatureExtractor




				def 					_snake_case  (self		):
										__lowerCAmelCase							       =				ASTFeatureExtractionTester(self		)




				def 					_snake_case  (self		):
										# Tests that all call wrap to encode_plus and batch_encode_plus
										__lowerCAmelCase							       =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
										# create three inputs of length 800, 1000, and 1200
										__lowerCAmelCase							       =				[floats_list((1, x)		)[0] for x in range(8_00       ,			14_00       ,			2_00		)]
										__lowerCAmelCase							       =				[np.asarray(__lowercase		) for speech_input in speech_inputs]

										# Test not batched input
										__lowerCAmelCase							       =				feat_extract(speech_inputs[0]       ,			return_tensors='''np'''		).input_values
										__lowerCAmelCase							       =				feat_extract(np_speech_inputs[0]       ,			return_tensors='''np'''		).input_values
										self.assertTrue(np.allclose(__lowercase       ,			__lowercase       ,			atol=1e-3		)		)

										# Test batched
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			padding=__lowercase       ,			return_tensors='''np'''		).input_values
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			padding=__lowercase       ,			return_tensors='''np'''		).input_values
										for enc_seq_a, enc_seq_a in zip(__lowercase       ,			__lowercase		):
																self.assertTrue(np.allclose(__lowercase       ,			__lowercase       ,			atol=1e-3		)		)

										# Test 2-D numpy arrays are batched.
										__lowerCAmelCase							       =				[floats_list((1, x)		)[0] for x in (8_00, 8_00, 8_00)]
										__lowerCAmelCase							       =				np.asarray(__lowercase		)
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			return_tensors='''np'''		).input_values
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			return_tensors='''np'''		).input_values
										for enc_seq_a, enc_seq_a in zip(__lowercase       ,			__lowercase		):
																self.assertTrue(np.allclose(__lowercase       ,			__lowercase       ,			atol=1e-3		)		)




				@require_torch
				def 					_snake_case  (self		):
										import torch

										__lowerCAmelCase							       =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
										__lowerCAmelCase							       =				np.random.rand(1_00		).astype(np.floataa		)
										__lowerCAmelCase							       =				np_speech_inputs.tolist()

										for inputs in [py_speech_inputs, np_speech_inputs]:
																__lowerCAmelCase							       =				feature_extractor.pad([{'''input_values''': inputs}]       ,			return_tensors='''np'''		)
																self.assertTrue(np_processed.input_values.dtype == np.floataa		)
																__lowerCAmelCase							       =				feature_extractor.pad([{'''input_values''': inputs}]       ,			return_tensors='''pt'''		)
																self.assertTrue(pt_processed.input_values.dtype == torch.floataa		)




				def 					_snake_case  (self       ,			__lowercase		):
										from datasets import load_dataset

										__lowerCAmelCase							       =				load_dataset('''hf-internal-testing/librispeech_asr_dummy'''       ,			'''clean'''       ,			split='''validation'''		)
										# automatic decoding with librispeech
										__lowerCAmelCase							       =				ds.sort('''id'''		).select(range(__lowercase		)		)[:num_samples]['''audio''']

										return [x["array"] for x in speech_samples]




				@require_torch
				def 					_snake_case  (self		):
										# fmt: off
										__lowerCAmelCase							       =				torch.tensor(
										    [-0.9_8_9_4, -1.2_7_7_6, -0.9_0_6_6, -1.2_7_7_6, -0.9_3_4_9, -1.2_6_0_9, -1.0_3_8_6, -1.2_7_7_6,
										     -1.1_5_6_1, -1.2_7_7_6, -1.2_0_5_2, -1.2_7_2_3, -1.2_1_9_0, -1.2_1_3_2, -1.2_7_7_6, -1.1_1_3_3,
										     -1.1_9_5_3, -1.1_3_4_3, -1.1_5_8_4, -1.2_2_0_3, -1.1_7_7_0, -1.2_4_7_4, -1.2_3_8_1, -1.1_9_3_6,
										     -0.9_2_7_0, -0.8_3_1_7, -0.8_0_4_9, -0.7_7_0_6, -0.7_5_6_5, -0.7_8_6_9]		)
										# fmt: on

										__lowerCAmelCase							       =				self._load_datasamples(1		)
										__lowerCAmelCase							       =				ASTFeatureExtractor()
										__lowerCAmelCase							       =				feature_extractor(__lowercase       ,			return_tensors='''pt'''		).input_values
										self.assertEquals(input_values.shape       ,			(1, 10_24, 1_28)		)
										self.assertTrue(torch.allclose(input_values[0, 0, :30]       ,			__lowercase       ,			atol=1e-4		)		)

