
'''simple docstring'''



from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


lowerCamelCase	 =       logging.get_logger(__name__)





class _UpperCamelCase		(     _UpperCAmelCase		):





				'''simple docstring'''





				lowerCAmelCase__					    =    ["""input_features""", """attention_mask"""]

				def __init__(							self  :  Optional[Any]  ,       _lowerCAmelCase  :  int=8_0  ,       _lowerCAmelCase  :  Tuple=1_6_0_0_0  ,       _lowerCAmelCase  :  int=8_0  ,       _lowerCAmelCase  :  int=0.0  ,       _lowerCAmelCase  :  Any=True  ,       _lowerCAmelCase  :  int=True  ,       _lowerCAmelCase  :  List[Any]=True  ,       **_lowerCAmelCase  :  Optional[Any]  ,       ):







											'''simple docstring'''

											super().__init__(feature_size=lowerCamelCase_  ,       sampling_rate=lowerCamelCase_  ,       padding_value=lowerCamelCase_  ,       **lowerCamelCase_)
											__lowercase       	=num_mel_bins
											__lowercase       	=do_ceptral_normalize
											__lowercase       	=normalize_means
											__lowercase       	=normalize_vars
											__lowercase       	=True

				def 				__lowerCamelCase     (							self  :  Union[str, Any]  ,       _lowerCAmelCase  :  np.ndarray  ,       ):







											'''simple docstring'''

											__lowercase       	=waveform * (2**1_5)  # Kaldi compliance: 16-bit signed integers
											__lowercase       	=torch.from_numpy(lowerCamelCase_).unsqueeze(0)
											__lowercase       	=ta_kaldi.fbank(lowerCamelCase_  ,       num_mel_bins=self.num_mel_bins  ,       sample_frequency=self.sampling_rate)
											return features.numpy()

				@staticmethod
				def 				__lowerCamelCase     (							_lowerCAmelCase  :  np.ndarray  ,       _lowerCAmelCase  :  int  ,       _lowerCAmelCase  :  Optional[bool] = True  ,       _lowerCAmelCase  :  Optional[bool] = True  ,       _lowerCAmelCase  :  float = 0.0  ,       ):







											'''simple docstring'''

											if normalize_means:
																		__lowercase       	=x[:input_length].mean(axis=0)
																		__lowercase       	=np.subtract(lowerCamelCase_  ,       lowerCamelCase_)
											if normalize_vars:
																		__lowercase       	=x[:input_length].std(axis=0)
																		__lowercase       	=np.divide(lowerCamelCase_  ,       lowerCamelCase_)

											if input_length < x.shape[0]:
																		__lowercase       	=padding_value

											# make sure array is in float32
											__lowercase       	=x.astype(np.floataa)

											return x

				def 				__lowerCamelCase     (							self  :  Tuple  ,       _lowerCAmelCase  :  List[np.ndarray]  ,       _lowerCAmelCase  :  Optional[np.ndarray] = None):







											'''simple docstring'''

											__lowercase       	=attention_mask.sum(-1) if attention_mask is not None else [x.shape[0] for x in input_features]
											return [
											    self.utterance_cmvn(lowerCamelCase_  ,       lowerCamelCase_  ,       self.normalize_means  ,       self.normalize_vars  ,       self.padding_value)
											    for x, n in zip(lowerCamelCase_  ,       lowerCamelCase_)
											]

				def __call__(							self  :  int  ,       _lowerCAmelCase  :  Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]  ,       _lowerCAmelCase  :  Union[bool, str, PaddingStrategy] = False  ,       _lowerCAmelCase  :  Optional[int] = None  ,       _lowerCAmelCase  :  bool = False  ,       _lowerCAmelCase  :  Optional[int] = None  ,       _lowerCAmelCase  :  Optional[Union[str, TensorType]] = None  ,       _lowerCAmelCase  :  Optional[int] = None  ,       _lowerCAmelCase  :  Optional[bool] = None  ,       **_lowerCAmelCase  :  str  ,       ):







											'''simple docstring'''

											if sampling_rate is not None:
																		if sampling_rate != self.sampling_rate:
																									raise ValueError(
																									    f"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
																									    f""" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"""
																									    f""" {self.sampling_rate} and not {sampling_rate}.""")
											else:
																		logger.warning(
																		    'It is strongly recommended to pass the `sampling_rate` argument to this function. '
																		    'Failing to do so can result in silent errors that might be hard to debug.')

											__lowercase       	=isinstance(lowerCamelCase_  ,       np.ndarray) and len(raw_speech.shape) > 1
											if is_batched_numpy and len(raw_speech.shape) > 2:
																		raise ValueError(f"""Only mono-channel audio is supported for input to {self}""")
											__lowercase       	=is_batched_numpy or (
											    isinstance(lowerCamelCase_  ,       (list, tuple)) and (isinstance(raw_speech[0]  ,       (np.ndarray, tuple, list)))
											)

											if is_batched:
																		__lowercase       	=[np.asarray(lowerCamelCase_  ,       dtype=np.floataa) for speech in raw_speech]
											elif not is_batched and not isinstance(lowerCamelCase_  ,       np.ndarray):
																		__lowercase       	=np.asarray(lowerCamelCase_  ,       dtype=np.floataa)
											elif isinstance(lowerCamelCase_  ,       np.ndarray) and raw_speech.dtype is np.dtype(np.floataa):
																		__lowercase       	=raw_speech.astype(np.floataa)

											# always return batch
											if not is_batched:
																		__lowercase       	=[raw_speech]

											# extract fbank features
											__lowercase       	=[self._extract_fbank_features(lowerCamelCase_) for waveform in raw_speech]

											# convert into correct format for padding
											__lowercase       	=BatchFeature({'input_features': features})

											__lowercase       	=self.pad(
											    lowerCamelCase_  ,       padding=lowerCamelCase_  ,       max_length=lowerCamelCase_  ,       truncation=lowerCamelCase_  ,       pad_to_multiple_of=lowerCamelCase_  ,       return_attention_mask=lowerCamelCase_  ,       **lowerCamelCase_  ,       )

											# make sure list is in array format
											__lowercase       	=padded_inputs.get('input_features')
											if isinstance(input_features[0]  ,       lowerCamelCase_):
																		__lowercase       	=[np.asarray(lowerCamelCase_  ,       dtype=np.floataa) for feature in input_features]

											__lowercase       	=padded_inputs.get('attention_mask')
											if attention_mask is not None:
																		__lowercase       	=[np.asarray(lowerCamelCase_  ,       dtype=np.intaa) for array in attention_mask]

											# Utterance-level cepstral mean and variance normalization
											if self.do_ceptral_normalize:
																		__lowercase       	=(
																		    np.array(lowerCamelCase_  ,       dtype=np.intaa)
																		    if self._get_padding_strategies(lowerCamelCase_  ,       max_length=lowerCamelCase_) is not PaddingStrategy.DO_NOT_PAD
																		    else None
																		)
																		__lowercase       	=self.normalize(
																		    padded_inputs['input_features']  ,       attention_mask=lowerCamelCase_)

											if return_tensors is not None:
																		__lowercase       	=padded_inputs.convert_to_tensors(lowerCamelCase_)

											return padded_inputs








'''simple docstring'''



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)

lowerCamelCase	 =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase	 =       {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase	 =       {
    """Salesforce/codegen-350M-mono""": 2048,
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    ["""input_ids""", """attention_mask"""]
				lowerCAmelCase__					    =    CodeGenTokenizer

				def __init__(							self  :  int  ,       _lowerCAmelCase  :  str=None  ,       _lowerCAmelCase  :  List[Any]=None  ,       _lowerCAmelCase  :  Optional[Any]=None  ,       _lowerCAmelCase  :  List[str]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[Any]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]=False  ,       **_lowerCAmelCase  :  Union[str, Any]  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       bos_token=_lowerCAmelCase  ,       eos_token=_lowerCAmelCase  ,       add_prefix_space=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if kwargs.pop('add_bos_token'  ,       _lowerCAmelCase):
																		__lowercase       	=kwargs.pop('name_or_path'  ,       '')
																		raise ValueError(
																		    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
																		    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
																		    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
																		    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
																		    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
																		    ' so that the fast tokenizer works correctly.')

											__lowercase       	=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																		__lowercase       	=getattr(_lowerCAmelCase  ,       pre_tok_state.pop('type'))
																		__lowercase       	=add_prefix_space
																		__lowercase       	=pre_tok_class(**_lowerCAmelCase)

											__lowercase       	=add_prefix_space

				def 				__lowerCamelCase     (							self  :  Any  ,       *_lowerCAmelCase  :  Tuple  ,       **_lowerCAmelCase  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)
											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[str]  ,       *_lowerCAmelCase  :  List[Any]  ,       **_lowerCAmelCase  :  List[str]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]  ,       _lowerCAmelCase  :  bool = False  ,       _lowerCAmelCase  :  bool = None  ,       _lowerCAmelCase  :  Optional[List[str]] = None  ,       **_lowerCAmelCase  :  Any  ,       ):







											'''simple docstring'''

											__lowercase       	=super().decode(
											    token_ids=_lowerCAmelCase  ,       skip_special_tokens=_lowerCAmelCase  ,       clean_up_tokenization_spaces=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if truncate_before_pattern is not None and len(_lowerCAmelCase) > 0:
																		__lowercase       	=self.truncate(_lowerCAmelCase  ,       _lowerCAmelCase)

											return decoded_text

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											def find_re(_lowerCAmelCase  :  List[Any]  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  str):
																		__lowercase       	=pattern.search(_lowerCAmelCase  ,       _lowerCAmelCase)
																		return m.start() if m else -1

											__lowercase       	=[re.compile(_lowerCAmelCase  ,       re.MULTILINE) for pattern in truncate_before_pattern]

											__lowercase       	=list(re.finditer('^print'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: prints[1].start()]

											__lowercase       	=list(re.finditer('^def'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: defs[1].start()]

											__lowercase       	=0

											__lowercase       	=[
											    pos for pos in [find_re(_lowerCAmelCase  ,       _lowerCAmelCase  ,       _lowerCAmelCase) for terminal in terminals] if pos != -1
											]

											if len(_lowerCAmelCase) > 0:
																		return completion[: min(_lowerCAmelCase)]
											else:
																		return completion



