





'''simple docstring'''







from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


_lowercase       						=			logging.get_logger(__name__)
class 			_lowercase						(       __a					):







					def __init__( self    ,   A__    ,   A__    ,   A__    ,   **A__    ) ->	Union[str, Any]:
						snake_case				=			feature_size
						snake_case				=			sampling_rate
						snake_case				=			padding_value

						snake_case				=			kwargs.pop('''padding_side'''    ,   '''right'''    )
						snake_case				=			kwargs.pop('''return_attention_mask'''    ,   A__    )

						super().__init__(**A__    )







					def     UpperCamelCase			( self    ,   A__    ,   A__ = True    ,   A__ = None    ,   A__ = False    ,   A__ = None    ,   A__ = None    ,   A__ = None    ,   ) ->	BatchFeature:
						# If we have a list of dicts, let's convert it in a dict of lists
						# We do this to allow using this method as a collate_fn function in PyTorch Dataloader
						if isinstance(A__    ,   (list, tuple)    ) and isinstance(processed_features[0]    ,   (dict, BatchFeature)    ):
							snake_case				=			{
							    key: [example[key] for example in processed_features] for key in processed_features[0].keys()
							}

						# The model's main input name, usually `input_values`, has be passed for padding
						if self.model_input_names[0] not in processed_features:
							raise ValueError(
							    '''You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`'''
							    F""" to this method that includes {self.model_input_names[0]}, but you provided"""
							    F""" {list(processed_features.keys()    )}"""    )

						snake_case				=			processed_features[self.model_input_names[0]]
						snake_case				=			(
						    return_attention_mask if return_attention_mask is not None else self.return_attention_mask
						)

						if len(A__    ) == 0:
							if return_attention_mask:
								snake_case				=			[]
							return processed_features

						# If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
						# and rebuild them afterwards if no return_tensors is specified
						# Note that we lose the specific device the tensor may be on for PyTorch

						snake_case				=			required_input[0]
						if isinstance(A__    ,   (list, tuple)    ):
							# first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
							snake_case				=			0
							while len(required_input[index]    ) == 0:
								index += 1
							if index < len(A__    ):
								snake_case				=			required_input[index][0]

						if return_tensors is None:
							if is_tf_tensor(A__    ):
								snake_case				=			'''tf'''
							elif is_torch_tensor(A__    ):
								snake_case				=			'''pt'''
							elif isinstance(A__    ,   (int, float, list, tuple, np.ndarray)    ):
								snake_case				=			'''np'''
							else:
								raise ValueError(
								    F"""type of {first_element} unknown: {type(A__    )}. """
								    '''Should be one of a python, numpy, pytorch or tensorflow object.'''    )

						for key, value in processed_features.items():
							if isinstance(value[0]    ,   (int, float)    ):
								snake_case				=			to_numpy(A__    )
							else:
								snake_case				=			[to_numpy(A__    ) for v in value]

								# Convert padding_strategy in PaddingStrategy
						snake_case				=			self._get_padding_strategies(padding=A__    ,   max_length=A__    )

						snake_case				=			processed_features[self.model_input_names[0]]

						snake_case				=			len(A__    )
						if not all(len(A__    ) == batch_size for v in processed_features.values()    ):
							raise ValueError('''Some items in the output dictionary have a different batch size than others.'''    )

						snake_case				=			[]
						for i in range(A__    ):
							snake_case				=			{k: v[i] for k, v in processed_features.items()}
							# truncation
							snake_case				=			self._truncate(
							    A__    ,   max_length=A__    ,   pad_to_multiple_of=A__    ,   truncation=A__    ,   )
							truncated_inputs.append(A__    )

						if padding_strategy == PaddingStrategy.LONGEST:
							# make sure that `max_length` cannot be longer than the longest truncated length
							snake_case				=			max(len(input_slice[self.model_input_names[0]]    ) for input_slice in truncated_inputs    )
							snake_case				=			PaddingStrategy.MAX_LENGTH

						snake_case				=			{}
						for i in range(A__    ):
							# padding
							snake_case				=			self._pad(
							    truncated_inputs[i]    ,   max_length=A__    ,   padding_strategy=A__    ,   pad_to_multiple_of=A__    ,   return_attention_mask=A__    ,   )

							for key, value in outputs.items():
								if key not in batch_outputs:
									snake_case				=			[]
								if value.dtype is np.dtype(np.floataa    ):
									snake_case				=			value.astype(np.floataa    )
								batch_outputs[key].append(A__    )

						return BatchFeature(A__    ,   tensor_type=A__    )







					def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ,   A__ = PaddingStrategy.DO_NOT_PAD    ,   A__ = None    ,   A__ = None    ,   ) ->	dict:
						snake_case				=			processed_features[self.model_input_names[0]]

						if padding_strategy == PaddingStrategy.LONGEST:
							snake_case				=			len(A__    )

						if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
							snake_case				=			((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

						snake_case				=			padding_strategy != PaddingStrategy.DO_NOT_PAD and len(A__    ) < max_length

						if return_attention_mask and "attention_mask" not in processed_features:
							snake_case				=			np.ones(len(A__    )    ,   dtype=np.intaa    )

						if needs_to_be_padded:
							snake_case				=			max_length - len(A__    )
							if self.padding_side == "right":
								if return_attention_mask:
									snake_case				=			np.pad(
									    processed_features['''attention_mask''']    ,   (0, difference)    )
								snake_case				=			((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
								snake_case				=			np.pad(
								    A__    ,   A__    ,   '''constant'''    ,   constant_values=self.padding_value    )
							elif self.padding_side == "left":
								if return_attention_mask:
									snake_case				=			np.pad(
									    processed_features['''attention_mask''']    ,   (difference, 0)    )
								snake_case				=			((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
								snake_case				=			np.pad(
								    A__    ,   A__    ,   '''constant'''    ,   constant_values=self.padding_value    )
							else:
								raise ValueError('''Invalid padding strategy:''' + str(self.padding_side    )    )

						return processed_features







					def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ,   A__ = None    ,   A__ = None    ,   ) ->	Union[str, Any]:
						if not truncation:
							return processed_features
						elif truncation and max_length is None:
							raise ValueError('''When setting ``truncation=True``, make sure that ``max_length`` is defined.'''    )

						snake_case				=			processed_features[self.model_input_names[0]]

						# find `max_length` that fits `pad_to_multiple_of`
						if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
							snake_case				=			((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

						snake_case				=			len(A__    ) > max_length

						if needs_to_be_truncated:
							snake_case				=			processed_features[self.model_input_names[0]][:max_length]
							if "attention_mask" in processed_features:
								snake_case				=			processed_features['''attention_mask'''][:max_length]

						return processed_features







					def     UpperCamelCase			( self    ,   A__=False    ,   A__=None    ) ->	Union[str, Any]:

						# Get padding strategy
						if padding is not False:
							if padding is True:
								snake_case				=			PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
							elif not isinstance(A__    ,   A__    ):
								snake_case				=			PaddingStrategy(A__    )
							elif isinstance(A__    ,   A__    ):
								snake_case				=			padding
						else:
							snake_case				=			PaddingStrategy.DO_NOT_PAD

						# Set max length if needed
						if max_length is None:
							if padding_strategy == PaddingStrategy.MAX_LENGTH:
								raise ValueError(
								    F"""When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"""    )

								# Test if we have a padding value
						if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
							raise ValueError(
							    '''Asking to pad but the feature_extractor does not have a padding value. Please select a value to use'''
							    ''' as `padding_value`. For example: `feature_extractor.padding_value = 0.0`.'''    )

						return padding_strategy








'''simple docstring'''







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_lowercase						(       __a				,		unittest.TestCase					):
     _UpperCAmelCase										=					CodeGenTokenizer
     _UpperCAmelCase										=					CodeGenTokenizerFast
     _UpperCAmelCase										=					True
     _UpperCAmelCase										=					{'''add_prefix_space''': True}
     _UpperCAmelCase										=					False







     def     UpperCamelCase			( self    ) ->	Tuple:
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      snake_case				=			[
          '''l''',
          '''o''',
          '''w''',
          '''e''',
          '''r''',
          '''s''',
          '''t''',
          '''i''',
          '''d''',
          '''n''',
          '''\u0120''',
          '''\u0120l''',
          '''\u0120n''',
          '''\u0120lo''',
          '''\u0120low''',
          '''er''',
          '''\u0120lowest''',
          '''\u0120newer''',
          '''\u0120wider''',
          '''<unk>''',
          '''<|endoftext|>''',
      ]
      snake_case				=			dict(zip(A__    ,   range(len(A__    )    )    )    )
      snake_case				=			['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
      snake_case				=			{'''unk_token''': '''<unk>'''}

      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''vocab_file''']    )
      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''merges_file''']    )
      with open(self.vocab_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write(json.dumps(A__    ) + '''\n'''    )
      with open(self.merges_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write('''\n'''.join(A__    )    )







     def     UpperCamelCase			( self    ,   **A__    ) ->	Union[str, Any]:
      kwargs.update(self.special_tokens_map    )
      return CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   **A__    )







     def     UpperCamelCase			( self    ,   **A__    ) ->	Union[str, Any]:
      kwargs.update(self.special_tokens_map    )
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname    ,   **A__    )







     def     UpperCamelCase			( self    ,   A__    ) ->	Tuple:
      snake_case				=			'''lower newer'''
      snake_case				=			'''lower newer'''
      return input_text, output_text







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				=			CodeGenTokenizer(self.vocab_file    ,   self.merges_file    ,   **self.special_tokens_map    )
      snake_case				=			'''lower newer'''
      snake_case				=			['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
      snake_case				=			tokenizer.tokenize(A__    ,   add_prefix_space=A__    )
      self.assertListEqual(A__    ,   A__    )

      snake_case				=			tokens + [tokenizer.unk_token]
      snake_case				=			[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(A__    )    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Optional[int]:
      if not self.test_rust_tokenizer:
       return

      snake_case				=			self.get_tokenizer()
      snake_case				=			self.get_rust_tokenizer(add_prefix_space=A__    )

      snake_case				=			'''lower newer'''

      # Testing tokenization
      snake_case				=			tokenizer.tokenize(A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.tokenize(A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing conversion to ids without special tokens
      snake_case				=			tokenizer.encode(A__    ,   add_special_tokens=A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.encode(A__    ,   add_special_tokens=A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing conversion to ids with special tokens
      snake_case				=			self.get_rust_tokenizer(add_prefix_space=A__    )
      snake_case				=			tokenizer.encode(A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.encode(A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing the unknown token
      snake_case				=			tokens + [rust_tokenizer.unk_token]
      snake_case				=			[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A__    )    ,   A__    )







     def     UpperCamelCase			( self    ,   *A__    ,   **A__    ) ->	List[str]:
      # It's very difficult to mix/test pretokenization with byte-level
      # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
      pass







     def     UpperCamelCase			( self    ,   A__=15    ) ->	Tuple:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
        snake_case				=			self.rust_tokenizer_class.from_pretrained(A__    ,   **A__    )

        # Simple input
        snake_case				=			'''This is a simple input'''
        snake_case				=			['''This is a simple input 1''', '''This is a simple input 2''']
        snake_case				=			('''This is a simple input''', '''This is a pair''')
        snake_case				=			[
            ('''This is a simple input 1''', '''This is a simple input 2'''),
            ('''This is a simple pair 1''', '''This is a simple pair 2'''),
        ]

        # Simple input tests
        self.assertRaises(A__    ,   tokenizer_r.encode    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Simple input
        self.assertRaises(A__    ,   tokenizer_r.encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Simple input
        self.assertRaises(
            A__    ,   tokenizer_r.batch_encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    ,   )

        # Pair input
        self.assertRaises(A__    ,   tokenizer_r.encode    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Pair input
        self.assertRaises(A__    ,   tokenizer_r.encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Pair input
        self.assertRaises(
            A__    ,   tokenizer_r.batch_encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    ,   )







     def     UpperCamelCase			( self    ) ->	Tuple:
      snake_case				=			CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   pad_token='''<pad>'''    )

      # Simple input
      snake_case				=			'''This is a simple input'''
      snake_case				=			['''This is a simple input looooooooong''', '''This is a simple input''']
      snake_case				=			('''This is a simple input''', '''This is a pair''')
      snake_case				=			[
          ('''This is a simple input loooooong''', '''This is a simple input'''),
          ('''This is a simple pair loooooong''', '''This is a simple pair'''),
      ]

      snake_case				=			tokenizer.pad_token_id

      snake_case				=			tokenizer(A__    ,   padding='''max_length'''    ,   max_length=30    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(A__    ,   padding=A__    ,   truncate=A__    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(*A__    ,   padding='''max_length'''    ,   max_length=60    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(A__    ,   padding=A__    ,   truncate=A__    ,   return_tensors='''np'''    )

      # s
      # test single string max_length padding
      self.assertEqual(out_s['''input_ids'''].shape[-1]    ,   30    )
      self.assertTrue(pad_token_id in out_s['''input_ids''']    )
      self.assertTrue(0 in out_s['''attention_mask''']    )

      # s2
      # test automatic padding
      self.assertEqual(out_sa['''input_ids'''].shape[-1]    ,   33    )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]    )
      self.assertFalse(0 in out_sa['''attention_mask'''][0]    )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]    )
      self.assertTrue(0 in out_sa['''attention_mask'''][1]    )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p['''input_ids'''].shape[-1]    ,   60    )
      self.assertTrue(pad_token_id in out_p['''input_ids''']    )
      self.assertTrue(0 in out_p['''attention_mask''']    )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa['''input_ids'''].shape[-1]    ,   52    )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]    )
      self.assertFalse(0 in out_pa['''attention_mask'''][0]    )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]    )
      self.assertTrue(0 in out_pa['''attention_mask'''][1]    )







     def     UpperCamelCase			( self    ) ->	str:
      snake_case				=			'''$$$'''
      snake_case				=			CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   bos_token=A__    ,   add_bos_token=A__    )

      snake_case				=			'''This is a simple input'''
      snake_case				=			['''This is a simple input 1''', '''This is a simple input 2''']

      snake_case				=			tokenizer.bos_token_id

      snake_case				=			tokenizer(A__    )
      snake_case				=			tokenizer(A__    )

      self.assertEqual(out_s.input_ids[0]    ,   A__    )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids    )    )

      snake_case				=			tokenizer.decode(out_s.input_ids    )
      snake_case				=			tokenizer.batch_decode(out_sa.input_ids    )

      self.assertEqual(decode_s.split()[0]    ,   A__    )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa    )    )







     @slow
     def     UpperCamelCase			( self    ) ->	Any:
      snake_case				=			CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''    )

      snake_case				=			'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
      snake_case				=			'''\nif len_a > len_b:      result = a\nelse:      result = b'''

      snake_case				=			tokenizer.encode(A__    )
      snake_case				=			['''^#''', re.escape('''<|endoftext|>'''    ), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
      snake_case				=			tokenizer.decode(A__    ,   truncate_before_pattern=A__    )
      self.assertEqual(A__    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      pass

