





'''simple docstring'''







from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


_lowercase       						=			logging.get_logger(__name__)
class 			_lowercase						(       __a					):
     _UpperCAmelCase										=					['''input_features''', '''attention_mask''']







     def __init__( self    ,   A__=80    ,   A__=1_60_00    ,   A__=0.0    ,   A__=10    ,   A__=25    ,   A__="hamming_window"    ,   A__=3_2_7_6_8.0    ,   A__=0.9_7    ,   A__=1.0    ,   A__=True    ,   A__=True    ,   A__=False    ,   **A__    ,   ) ->	Union[str, Any]:
      super().__init__(feature_size=A__    ,   sampling_rate=A__    ,   padding_value=A__    ,   **A__    )

      snake_case				=			feature_size
      snake_case				=			sampling_rate
      snake_case				=			padding_value
      snake_case				=			hop_length
      snake_case				=			win_length
      snake_case				=			frame_signal_scale
      snake_case				=			preemphasis_coeff
      snake_case				=			mel_floor
      snake_case				=			normalize_means
      snake_case				=			normalize_vars
      snake_case				=			win_function
      snake_case				=			return_attention_mask

      snake_case				=			win_length * sampling_rate // 10_00
      snake_case				=			hop_length * sampling_rate // 10_00

      snake_case				=			optimal_fft_length(self.sample_size    )
      snake_case				=			(self.n_fft // 2) + 1







     def     UpperCamelCase			( self    ,   A__    ) ->	np.ndarray:
      if self.win_function == "hamming_window":
       snake_case				=			window_function(window_length=self.sample_size    ,   name=self.win_function    ,   periodic=A__    )
      else:
       snake_case				=			window_function(window_length=self.sample_size    ,   name=self.win_function    )

      snake_case				=			mel_filter_bank(
          num_frequency_bins=self.n_freqs    ,   num_mel_filters=self.feature_size    ,   min_frequency=0.0    ,   max_frequency=self.sampling_rate / 2.0    ,   sampling_rate=self.sampling_rate    ,   )

      snake_case				=			spectrogram(
          one_waveform * self.frame_signal_scale    ,   window=A__    ,   frame_length=self.sample_size    ,   hop_length=self.sample_stride    ,   fft_length=self.n_fft    ,   center=A__    ,   preemphasis=self.preemphasis_coeff    ,   mel_filters=A__    ,   mel_floor=self.mel_floor    ,   log_mel='''log'''    ,   )
      return msfc_features.T







     def     UpperCamelCase			( self    ,   A__    ,   A__    ,   A__    ) ->	Optional[Any]:
      # make sure we normalize float32 arrays
      if self.normalize_means:
       snake_case				=			x[:input_length].mean(axis=0    )
       snake_case				=			np.subtract(A__    ,   A__    )
      if self.normalize_vars:
       snake_case				=			x[:input_length].std(axis=0    )
       snake_case				=			np.divide(A__    ,   A__    )

      if input_length < x.shape[0]:
       snake_case				=			padding_value

      # make sure array is in float32
      snake_case				=			x.astype(np.floataa    )

      return x







     def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ) ->	List[np.ndarray]:
      snake_case				=			attention_mask.sum(-1    ) if attention_mask is not None else [x.shape[0] for x in input_features]
      return [self._normalize_one(A__    ,   A__    ,   self.padding_value    ) for x, n in zip(A__    ,   A__    )]







     def __call__( self    ,   A__    ,   A__ = False    ,   A__ = None    ,   A__ = False    ,   A__ = None    ,   A__ = None    ,   A__ = None    ,   A__ = None    ,   **A__    ,   ) ->	BatchFeature:

      if sampling_rate is not None:
       if sampling_rate != self.sampling_rate:
        raise ValueError(
            F"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
            F""" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"""
            F""" {self.sampling_rate} and not {sampling_rate}."""    )
      else:
       logger.warning(
           '''It is strongly recommended to pass the ``sampling_rate`` argument to this function. '''
           '''Failing to do so can result in silent errors that might be hard to debug.'''    )

      snake_case				=			isinstance(A__    ,   np.ndarray    ) and len(raw_speech.shape    ) > 1
      if is_batched_numpy and len(raw_speech.shape    ) > 2:
       raise ValueError(F"""Only mono-channel audio is supported for input to {self}"""    )
      snake_case				=			is_batched_numpy or (
          isinstance(A__    ,   (list, tuple)    ) and (isinstance(raw_speech[0]    ,   (np.ndarray, tuple, list)    ))
      )

      if is_batched:
       snake_case				=			[np.asarray(A__    ,   dtype=np.floataa    ) for speech in raw_speech]
      elif not is_batched and not isinstance(A__    ,   np.ndarray    ):
       snake_case				=			np.asarray(A__    ,   dtype=np.floataa    )
      elif isinstance(A__    ,   np.ndarray    ) and raw_speech.dtype is np.dtype(np.floataa    ):
       snake_case				=			raw_speech.astype(np.floataa    )

      # always return batch
      if not is_batched:
       snake_case				=			[raw_speech]

      # extract fbank features
      snake_case				=			[self._extract_mfsc_features(A__    ) for one_waveform in raw_speech]

      # convert into correct format for padding
      snake_case				=			BatchFeature({'''input_features''': features}    )

      snake_case				=			self.pad(
          A__    ,   padding=A__    ,   max_length=A__    ,   truncation=A__    ,   pad_to_multiple_of=A__    ,   return_attention_mask=A__    ,   **A__    ,   )
      # make sure list is in array format
      snake_case				=			padded_inputs.get('''input_features'''    )
      if isinstance(input_features[0]    ,   A__    ):
       snake_case				=			[np.asarray(A__    ,   dtype=np.floataa    ) for feature in input_features]

      snake_case				=			padded_inputs.get('''attention_mask'''    )
      if attention_mask is not None:
       snake_case				=			[np.asarray(A__    ,   dtype=np.intaa    ) for array in attention_mask]

      if self.normalize_means or self.normalize_vars:
       snake_case				=			(
           np.array(A__    ,   dtype=np.intaa    )
           if self._get_padding_strategies(A__    ,   max_length=A__    ) is not PaddingStrategy.DO_NOT_PAD
           and padding
           else None
       )
       snake_case				=			self.normalize(
           padded_inputs['''input_features''']    ,   attention_mask=A__    )

      if return_tensors is not None:
       snake_case				=			padded_inputs.convert_to_tensors(A__    )

      return padded_inputs








'''simple docstring'''







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_lowercase						(       __a				,		unittest.TestCase					):
     _UpperCAmelCase										=					CodeGenTokenizer
     _UpperCAmelCase										=					CodeGenTokenizerFast
     _UpperCAmelCase										=					True
     _UpperCAmelCase										=					{'''add_prefix_space''': True}
     _UpperCAmelCase										=					False







     def     UpperCamelCase			( self    ) ->	Tuple:
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      snake_case				=			[
          '''l''',
          '''o''',
          '''w''',
          '''e''',
          '''r''',
          '''s''',
          '''t''',
          '''i''',
          '''d''',
          '''n''',
          '''\u0120''',
          '''\u0120l''',
          '''\u0120n''',
          '''\u0120lo''',
          '''\u0120low''',
          '''er''',
          '''\u0120lowest''',
          '''\u0120newer''',
          '''\u0120wider''',
          '''<unk>''',
          '''<|endoftext|>''',
      ]
      snake_case				=			dict(zip(A__    ,   range(len(A__    )    )    )    )
      snake_case				=			['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
      snake_case				=			{'''unk_token''': '''<unk>'''}

      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''vocab_file''']    )
      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''merges_file''']    )
      with open(self.vocab_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write(json.dumps(A__    ) + '''\n'''    )
      with open(self.merges_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write('''\n'''.join(A__    )    )







     def     UpperCamelCase			( self    ,   **A__    ) ->	Union[str, Any]:
      kwargs.update(self.special_tokens_map    )
      return CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   **A__    )







     def     UpperCamelCase			( self    ,   **A__    ) ->	Union[str, Any]:
      kwargs.update(self.special_tokens_map    )
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname    ,   **A__    )







     def     UpperCamelCase			( self    ,   A__    ) ->	Tuple:
      snake_case				=			'''lower newer'''
      snake_case				=			'''lower newer'''
      return input_text, output_text







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				=			CodeGenTokenizer(self.vocab_file    ,   self.merges_file    ,   **self.special_tokens_map    )
      snake_case				=			'''lower newer'''
      snake_case				=			['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
      snake_case				=			tokenizer.tokenize(A__    ,   add_prefix_space=A__    )
      self.assertListEqual(A__    ,   A__    )

      snake_case				=			tokens + [tokenizer.unk_token]
      snake_case				=			[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(A__    )    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Optional[int]:
      if not self.test_rust_tokenizer:
       return

      snake_case				=			self.get_tokenizer()
      snake_case				=			self.get_rust_tokenizer(add_prefix_space=A__    )

      snake_case				=			'''lower newer'''

      # Testing tokenization
      snake_case				=			tokenizer.tokenize(A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.tokenize(A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing conversion to ids without special tokens
      snake_case				=			tokenizer.encode(A__    ,   add_special_tokens=A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.encode(A__    ,   add_special_tokens=A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing conversion to ids with special tokens
      snake_case				=			self.get_rust_tokenizer(add_prefix_space=A__    )
      snake_case				=			tokenizer.encode(A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.encode(A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing the unknown token
      snake_case				=			tokens + [rust_tokenizer.unk_token]
      snake_case				=			[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A__    )    ,   A__    )







     def     UpperCamelCase			( self    ,   *A__    ,   **A__    ) ->	List[str]:
      # It's very difficult to mix/test pretokenization with byte-level
      # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
      pass







     def     UpperCamelCase			( self    ,   A__=15    ) ->	Tuple:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
        snake_case				=			self.rust_tokenizer_class.from_pretrained(A__    ,   **A__    )

        # Simple input
        snake_case				=			'''This is a simple input'''
        snake_case				=			['''This is a simple input 1''', '''This is a simple input 2''']
        snake_case				=			('''This is a simple input''', '''This is a pair''')
        snake_case				=			[
            ('''This is a simple input 1''', '''This is a simple input 2'''),
            ('''This is a simple pair 1''', '''This is a simple pair 2'''),
        ]

        # Simple input tests
        self.assertRaises(A__    ,   tokenizer_r.encode    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Simple input
        self.assertRaises(A__    ,   tokenizer_r.encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Simple input
        self.assertRaises(
            A__    ,   tokenizer_r.batch_encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    ,   )

        # Pair input
        self.assertRaises(A__    ,   tokenizer_r.encode    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Pair input
        self.assertRaises(A__    ,   tokenizer_r.encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Pair input
        self.assertRaises(
            A__    ,   tokenizer_r.batch_encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    ,   )







     def     UpperCamelCase			( self    ) ->	Tuple:
      snake_case				=			CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   pad_token='''<pad>'''    )

      # Simple input
      snake_case				=			'''This is a simple input'''
      snake_case				=			['''This is a simple input looooooooong''', '''This is a simple input''']
      snake_case				=			('''This is a simple input''', '''This is a pair''')
      snake_case				=			[
          ('''This is a simple input loooooong''', '''This is a simple input'''),
          ('''This is a simple pair loooooong''', '''This is a simple pair'''),
      ]

      snake_case				=			tokenizer.pad_token_id

      snake_case				=			tokenizer(A__    ,   padding='''max_length'''    ,   max_length=30    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(A__    ,   padding=A__    ,   truncate=A__    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(*A__    ,   padding='''max_length'''    ,   max_length=60    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(A__    ,   padding=A__    ,   truncate=A__    ,   return_tensors='''np'''    )

      # s
      # test single string max_length padding
      self.assertEqual(out_s['''input_ids'''].shape[-1]    ,   30    )
      self.assertTrue(pad_token_id in out_s['''input_ids''']    )
      self.assertTrue(0 in out_s['''attention_mask''']    )

      # s2
      # test automatic padding
      self.assertEqual(out_sa['''input_ids'''].shape[-1]    ,   33    )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]    )
      self.assertFalse(0 in out_sa['''attention_mask'''][0]    )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]    )
      self.assertTrue(0 in out_sa['''attention_mask'''][1]    )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p['''input_ids'''].shape[-1]    ,   60    )
      self.assertTrue(pad_token_id in out_p['''input_ids''']    )
      self.assertTrue(0 in out_p['''attention_mask''']    )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa['''input_ids'''].shape[-1]    ,   52    )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]    )
      self.assertFalse(0 in out_pa['''attention_mask'''][0]    )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]    )
      self.assertTrue(0 in out_pa['''attention_mask'''][1]    )







     def     UpperCamelCase			( self    ) ->	str:
      snake_case				=			'''$$$'''
      snake_case				=			CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   bos_token=A__    ,   add_bos_token=A__    )

      snake_case				=			'''This is a simple input'''
      snake_case				=			['''This is a simple input 1''', '''This is a simple input 2''']

      snake_case				=			tokenizer.bos_token_id

      snake_case				=			tokenizer(A__    )
      snake_case				=			tokenizer(A__    )

      self.assertEqual(out_s.input_ids[0]    ,   A__    )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids    )    )

      snake_case				=			tokenizer.decode(out_s.input_ids    )
      snake_case				=			tokenizer.batch_decode(out_sa.input_ids    )

      self.assertEqual(decode_s.split()[0]    ,   A__    )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa    )    )







     @slow
     def     UpperCamelCase			( self    ) ->	Any:
      snake_case				=			CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''    )

      snake_case				=			'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
      snake_case				=			'''\nif len_a > len_b:      result = a\nelse:      result = b'''

      snake_case				=			tokenizer.encode(A__    )
      snake_case				=			['''^#''', re.escape('''<|endoftext|>'''    ), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
      snake_case				=			tokenizer.decode(A__    ,   truncate_before_pattern=A__    )
      self.assertEqual(A__    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      pass

