
"""simple docstring"""



import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin




class 		a_						( _UpperCAmelCase	):
    # to overwrite at feature extractactor specific tests
    a      :				Any     =				None
    a      :				str     =				None







    @property
    def      _snake_case					(			self			:       int     )		->Optional[Any]:






        '''simple docstring'''






        return self.feat_extract_tester.prepare_feat_extract_dict()







    def      _snake_case					(			self			:       Optional[Any]     )		->Optional[Any]:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        self.assertTrue(hasattr(__UpperCamelCase     ,					"""feature_size"""     )     )
        self.assertTrue(hasattr(__UpperCamelCase     ,					"""sampling_rate"""     )     )
        self.assertTrue(hasattr(__UpperCamelCase     ,					"""padding_value"""     )     )







    def      _snake_case					(			self			:       List[Any]     )		->int:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common()
        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     )

        self.assertTrue(all(len(__UpperCamelCase     ) == len(__UpperCamelCase     ) for x, y in zip(__UpperCamelCase     ,					processed_features[input_name]     )     )     )

        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common(equal_length=__UpperCamelCase     )
        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     ,					tensor_type="""np"""     )

        _UpperCAmelCase          =   processed_features[input_name]

        if len(batch_features_input.shape     ) < 3:
            _UpperCAmelCase          =   batch_features_input[:, :, None]

        self.assertTrue(
            batch_features_input.shape
            == (self.feat_extract_tester.batch_size, len(speech_inputs[0]     ), self.feat_extract_tester.feature_size)     )







    @require_torch
    def      _snake_case					(			self			:       List[Any]     )		->Any:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common(equal_length=__UpperCamelCase     )
        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     ,					tensor_type="""pt"""     )

        _UpperCAmelCase          =   processed_features[input_name]

        if len(batch_features_input.shape     ) < 3:
            _UpperCAmelCase          =   batch_features_input[:, :, None]

        self.assertTrue(
            batch_features_input.shape
            == (self.feat_extract_tester.batch_size, len(speech_inputs[0]     ), self.feat_extract_tester.feature_size)     )







    @require_tf
    def      _snake_case					(			self			:       Union[str, Any]     )		->str:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common(equal_length=__UpperCamelCase     )
        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     ,					tensor_type="""tf"""     )

        _UpperCAmelCase          =   processed_features[input_name]

        if len(batch_features_input.shape     ) < 3:
            _UpperCAmelCase          =   batch_features_input[:, :, None]

        self.assertTrue(
            batch_features_input.shape
            == (self.feat_extract_tester.batch_size, len(speech_inputs[0]     ), self.feat_extract_tester.feature_size)     )







    def      _snake_case					(			self			:       Tuple     ,					__UpperCamelCase			:       List[str]=False     )		->Union[str, Any]:






        '''simple docstring'''






        def _inputs_have_equal_length(__UpperCamelCase			:       Any     ):
            _UpperCAmelCase          =   len(input[0]     )
            for input_slice in input[1:]:
                if len(__UpperCamelCase     ) != length:
                    return False
            return True

        def _inputs_are_equal(__UpperCamelCase			:       Union[str, Any]     ,					__UpperCamelCase			:       int     ):
            if len(__UpperCamelCase     ) != len(__UpperCamelCase     ):
                return False

            for input_slice_a, input_slice_a in zip(__UpperCamelCase     ,					__UpperCamelCase     ):
                if not np.allclose(np.asarray(__UpperCamelCase     )     ,					np.asarray(__UpperCamelCase     )     ,					atol=1e-3     ):
                    return False
            return True

        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common(numpify=__UpperCamelCase     )
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     )

        _UpperCAmelCase          =   self.feat_extract_tester.seq_length_diff
        _UpperCAmelCase          =   self.feat_extract_tester.max_seq_length + pad_diff
        _UpperCAmelCase          =   self.feat_extract_tester.min_seq_length
        _UpperCAmelCase          =   self.feat_extract_tester.batch_size
        _UpperCAmelCase          =   self.feat_extract_tester.feature_size

        # test padding for List[int] + numpy
        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding=__UpperCamelCase     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[-1]     )     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					return_tensors="""np"""     )
        _UpperCAmelCase          =   input_a[input_name]

        # max_length parameter has to be provided when setting `padding="max_length"`
        with self.assertRaises(__UpperCamelCase     ):
            feat_extract.pad(__UpperCamelCase     ,					padding="""max_length"""     )[input_name]

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=__UpperCamelCase     ,					return_tensors="""np"""     )
        _UpperCAmelCase          =   input_a[input_name]

        self.assertFalse(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertTrue(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertTrue(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertTrue(_inputs_are_equal(__UpperCamelCase     ,					__UpperCamelCase     )     )
        self.assertTrue(len(input_a[0]     ) == pad_min_length     )
        self.assertTrue(len(input_a[1]     ) == pad_min_length + pad_diff     )
        self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0]     ))     )
        self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length)     )

        if feature_size > 1:
            self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size     )

        # test padding for `pad_to_multiple_of` for List[int] + numpy
        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					pad_to_multiple_of=10     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					pad_to_multiple_of=10     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					pad_to_multiple_of=10     ,					max_length=__UpperCamelCase     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					pad_to_multiple_of=10     ,					max_length=__UpperCamelCase     ,					return_tensors="""np"""     ,					)
        _UpperCAmelCase          =   input_a[input_name]

        self.assertTrue(all(len(__UpperCamelCase     ) % 10 == 0 for x in input_a     )     )
        self.assertTrue(_inputs_are_equal(__UpperCamelCase     ,					__UpperCamelCase     )     )

        _UpperCAmelCase          =   pad_max_length if pad_max_length % 10 == 0 else (pad_max_length // 10 + 1) * 10
        self.assertTrue(all(len(__UpperCamelCase     ) == expected_mult_pad_length for x in input_a     )     )
        self.assertEqual(input_a.shape[:2]     ,					(batch_size, expected_mult_pad_length)     )

        if feature_size > 1:
            self.assertTrue(input_a.shape[2] == feature_size     )

        # Check padding value is correct
        _UpperCAmelCase          =   (np.ones(self.feat_extract_tester.feature_size     ) * feat_extract.padding_value).sum()
        self.assertTrue(
            abs(np.asarray(input_a[0]     )[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)     )
            < 1e-3     )
        self.assertTrue(
            abs(
                np.asarray(input_a[1]     )[pad_min_length + pad_diff :].sum()
                - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff)     )
            < 1e-3     )
        self.assertTrue(
            abs(
                np.asarray(input_a[2]     )[pad_min_length + 2 * pad_diff :].sum()
                - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff)     )
            < 1e-3     )
        self.assertTrue(
            abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)     ) < 1e-3     )
        self.assertTrue(
            abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length)     )
            < 1e-3     )







    def      _snake_case					(			self			:       Dict     ,					__UpperCamelCase			:       str=False     )		->List[Any]:






        '''simple docstring'''






        def _inputs_have_equal_length(__UpperCamelCase			:       Union[str, Any]     ):
            _UpperCAmelCase          =   len(input[0]     )
            for input_slice in input[1:]:
                if len(__UpperCamelCase     ) != length:
                    return False
            return True

        def _inputs_are_equal(__UpperCamelCase			:       List[str]     ,					__UpperCamelCase			:       Union[str, Any]     ):
            if len(__UpperCamelCase     ) != len(__UpperCamelCase     ):
                return False

            for input_slice_a, input_slice_a in zip(__UpperCamelCase     ,					__UpperCamelCase     ):
                if not np.allclose(np.asarray(__UpperCamelCase     )     ,					np.asarray(__UpperCamelCase     )     ,					atol=1e-3     ):
                    return False
            return True

        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common(numpify=__UpperCamelCase     )
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     )

        # truncate to smallest
        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[0]     )     ,					truncation=__UpperCamelCase     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[0]     )     )
        _UpperCAmelCase          =   input_a[input_name]

        self.assertTrue(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertFalse(_inputs_have_equal_length(__UpperCamelCase     )     )

        # truncate to smallest with np
        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[0]     )     ,					return_tensors="""np"""     ,					truncation=__UpperCamelCase     ,					)
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[0]     )     ,					return_tensors="""np"""     )
        _UpperCAmelCase          =   input_a[input_name]

        self.assertTrue(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertTrue(input_a.shape[1] == len(speech_inputs[0]     )     )

        # since truncation forces padding to be smaller than longest input
        # function can't return `np.ndarray`, but has to return list
        self.assertFalse(_inputs_have_equal_length(__UpperCamelCase     )     )

        # truncate to middle
        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[1]     )     ,					truncation=__UpperCamelCase     ,					return_tensors="""np"""     ,					)
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[1]     )     ,					truncation=__UpperCamelCase     )
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[1]     )     ,					return_tensors="""np"""     )
        _UpperCAmelCase          =   input_a[input_name]

        self.assertTrue(input_a.shape[1] == len(speech_inputs[1]     )     )
        self.assertTrue(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertTrue(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertTrue(_inputs_are_equal(__UpperCamelCase     ,					__UpperCamelCase     )     )

        # since truncation forces padding to be smaller than longest input
        # function can't return `np.ndarray`, but has to return list
        self.assertFalse(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertTrue(len(input_a[-1]     ) == len(speech_inputs[-1]     )     )

        # padding has to be max_length when setting `truncation=True`
        with self.assertRaises(__UpperCamelCase     ):
            feat_extract.pad(__UpperCamelCase     ,					truncation=__UpperCamelCase     )[input_name]

        # padding has to be max_length when setting `truncation=True`
        with self.assertRaises(__UpperCamelCase     ):
            feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					truncation=__UpperCamelCase     )[input_name]

        # padding has to be max_length when setting `truncation=True`
        with self.assertRaises(__UpperCamelCase     ):
            feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					truncation=__UpperCamelCase     )[input_name]

        # max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
        with self.assertRaises(__UpperCamelCase     ):
            feat_extract.pad(__UpperCamelCase     ,					padding="""max_length"""     ,					truncation=__UpperCamelCase     )[input_name]

        # test truncation for `pad_to_multiple_of` for List[int] + numpy
        _UpperCAmelCase          =   12
        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[0]     )     ,					pad_to_multiple_of=__UpperCamelCase     ,					truncation=__UpperCamelCase     ,					)
        _UpperCAmelCase          =   input_a[input_name]

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=len(speech_inputs[0]     )     ,					pad_to_multiple_of=__UpperCamelCase     ,					)
        _UpperCAmelCase          =   input_a[input_name]

        # retrieve expected_length as multiple of pad_to_multiple_of
        _UpperCAmelCase          =   len(speech_inputs[0]     )
        if expected_length % pad_to_multiple_of != 0:
            _UpperCAmelCase          =   ((len(speech_inputs[0]     ) // pad_to_multiple_of) + 1) * pad_to_multiple_of

        self.assertTrue(len(input_a[0]     ) == expected_length     )
        self.assertTrue(_inputs_have_equal_length(__UpperCamelCase     )     )
        self.assertFalse(_inputs_have_equal_length(__UpperCamelCase     )     )







    def      _snake_case					(			self			:       List[Any]     )		->Union[str, Any]:






        '''simple docstring'''






        self._check_padding(numpify=__UpperCamelCase     )







    def      _snake_case					(			self			:       Dict     )		->Dict:






        '''simple docstring'''






        self._check_padding(numpify=__UpperCamelCase     )







    def      _snake_case					(			self			:       str     )		->Dict:






        '''simple docstring'''






        self._check_truncation(numpify=__UpperCamelCase     )







    def      _snake_case					(			self			:       List[str]     )		->Optional[Any]:






        '''simple docstring'''






        self._check_truncation(numpify=__UpperCamelCase     )







    @require_torch
    def      _snake_case					(			self			:       int     )		->Any:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common()
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     )

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					return_tensors="""np"""     )[input_name]
        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					return_tensors="""pt"""     )[input_name]

        self.assertTrue(abs(input_np.astype(np.floataa     ).sum() - input_pt.numpy().astype(np.floataa     ).sum()     ) < 1e-2     )







    @require_tf
    def      _snake_case					(			self			:       Dict     )		->int:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feature_extraction_class(**self.feat_extract_dict     )
        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common()
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     )

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					return_tensors="""np"""     )[input_name]
        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					return_tensors="""tf"""     )[input_name]

        self.assertTrue(abs(input_np.astype(np.floataa     ).sum() - input_tf.numpy().astype(np.floataa     ).sum()     ) < 1e-2     )







    def      _snake_case					(			self			:       Dict     )		->Union[str, Any]:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feat_extract_dict
        _UpperCAmelCase          =   True
        _UpperCAmelCase          =   self.feature_extraction_class(**__UpperCamelCase     )
        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common()
        _UpperCAmelCase          =   [len(__UpperCamelCase     ) for x in speech_inputs]
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     )

        _UpperCAmelCase          =   feat_extract.pad(__UpperCamelCase     ,					padding="""longest"""     ,					return_tensors="""np"""     )
        self.assertIn("""attention_mask"""     ,					__UpperCamelCase     )
        self.assertListEqual(list(processed.attention_mask.shape     )     ,					list(processed[input_name].shape[:2]     )     )
        self.assertListEqual(processed.attention_mask.sum(-1     ).tolist()     ,					__UpperCamelCase     )







    def      _snake_case					(			self			:       Any     )		->List[Any]:






        '''simple docstring'''






        _UpperCAmelCase          =   self.feat_extract_dict
        _UpperCAmelCase          =   True
        _UpperCAmelCase          =   self.feature_extraction_class(**__UpperCamelCase     )
        _UpperCAmelCase          =   self.feat_extract_tester.prepare_inputs_for_common()
        _UpperCAmelCase          =   [len(__UpperCamelCase     ) for x in speech_inputs]
        _UpperCAmelCase          =   feat_extract.model_input_names[0]

        _UpperCAmelCase          =   BatchFeature({input_name: speech_inputs}     )
        _UpperCAmelCase          =   min(__UpperCamelCase     )

        _UpperCAmelCase          =   feat_extract.pad(
            __UpperCamelCase     ,					padding="""max_length"""     ,					max_length=__UpperCamelCase     ,					truncation=__UpperCamelCase     ,					return_tensors="""np"""     )
        self.assertIn("""attention_mask"""     ,					__UpperCamelCase     )
        self.assertListEqual(
            list(processed_pad.attention_mask.shape     )     ,					[processed_pad[input_name].shape[0], max_length]     )
        self.assertListEqual(
            processed_pad.attention_mask[:, :max_length].sum(-1     ).tolist()     ,					[max_length for x in speech_inputs]     )

"""simple docstring"""



import re

from filelock import FileLock


try:
 import nltk

 a					:	str							=		True
except (ImportError, ModuleNotFoundError):
 a					:	List[str]							=		False

if NLTK_AVAILABLE:
 with FileLock('''.lock''') as lock:
  nltk.download('''punkt''', quiet=True)


def  _UpperCamelCase	(						_A      )    ->  str:





    """simple docstring"""

    re.sub("""<n>"""					,      """"""					,      _A      )  # remove pegasus newline char
    assert NLTK_AVAILABLE, "nltk must be installed to separate newlines between sentences. (pip install nltk)"
    return "\n".join(nltk.sent_tokenize(_A      )      )