






'''simple docstring'''






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__A    	=   logging.get_logger(__name__)

__A    	=   {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A    	=   {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

__A    	=   {
    '''Salesforce/codegen-350M-mono''': 2_0_4_8,
}




class a_    (    UpperCamelCase_							):
   _snake_case        =							VOCAB_FILES_NAMES
   _snake_case        =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case        =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case        =							["""input_ids""", """attention_mask"""]
   _snake_case        =							CodeGenTokenizer







   def __init__(self					,   __a=None					,   __a=None					,   __a=None					,   __a="<|endoftext|>"					,   __a="<|endoftext|>"					,   __a="<|endoftext|>"					,   __a=False					,   **__a					,   )			->      Dict:
      """simple docstring"""






      super().__init__(
          __a					,   __a					,   tokenizer_file=__a					,   unk_token=__a					,   bos_token=__a					,   eos_token=__a					,   add_prefix_space=__a					,   **__a					,   )

      if kwargs.pop('add_bos_token'					,   __a):
         __snake_case				:      List[Any]     							= kwargs.pop('name_or_path'					,   '')
         raise ValueError(
             'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
             'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
             F"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
             F"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
             'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
             ' so that the fast tokenizer works correctly.')

      __snake_case				:      Optional[Any]     							= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
      if pre_tok_state.get('add_prefix_space'					,   __a) != add_prefix_space:
         __snake_case				:      Dict     							= getattr(__a					,   pre_tok_state.pop('type'))
         __snake_case				:      Dict     							= add_prefix_space
         __snake_case				:      Any     							= pre_tok_class(**__a)

      __snake_case				:      Any     							= add_prefix_space







   def     SCREAMING_SNAKE_CASE__					(self					,   *__a					,   **__a)			->      BatchEncoding:
      """simple docstring"""






      __snake_case				:      List[str]     							= kwargs.get('is_split_into_words'					,   __a)
      assert self.add_prefix_space or not is_split_into_words, (
          F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*__a					,   **__a)







   def     SCREAMING_SNAKE_CASE__					(self					,   *__a					,   **__a)			->      BatchEncoding:
      """simple docstring"""






      __snake_case				:      Tuple     							= kwargs.get('is_split_into_words'					,   __a)

      assert self.add_prefix_space or not is_split_into_words, (
          F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*__a					,   **__a)







   def     SCREAMING_SNAKE_CASE__					(self					,   __a					,   __a = None)			->      Tuple[str]:
      """simple docstring"""






      __snake_case				:      int     							= self._tokenizer.model.save(__a					,   name=__a)
      return tuple(__a)







   def     SCREAMING_SNAKE_CASE__					(self					,   __a					,   __a = False					,   __a = None					,   __a = None					,   **__a					,   )			->      str:
      """simple docstring"""






      __snake_case				:      Dict     							= super().decode(
          token_ids=__a					,   skip_special_tokens=__a					,   clean_up_tokenization_spaces=__a					,   **__a					,   )

      if truncate_before_pattern is not None and len(__a) > 0:
         __snake_case				:      Tuple     							= self.truncate(__a					,   __a)

      return decoded_text







   def     SCREAMING_SNAKE_CASE__					(self					,   __a					,   __a)			->      str:
      """simple docstring"""






      def find_re(__a					,   __a					,   __a):
         __snake_case				:      List[Any]     							= pattern.search(__a					,   __a)
         return m.start() if m else -1

      __snake_case				:      Union[str, Any]     							= [re.compile(__a					,   re.MULTILINE) for pattern in truncate_before_pattern]

      __snake_case				:      List[str]     							= list(re.finditer('^print'					,   __a					,   re.MULTILINE))

      if len(__a) > 1:
         __snake_case				:      Tuple     							= completion[: prints[1].start()]

      __snake_case				:      Dict     							= list(re.finditer('^def'					,   __a					,   re.MULTILINE))

      if len(__a) > 1:
         __snake_case				:      Union[str, Any]     							= completion[: defs[1].start()]

      __snake_case				:      List[str]     							= 0

      __snake_case				:      Optional[int]     							= [
          pos for pos in [find_re(__a					,   __a					,   __a) for terminal in terminals] if pos != -1
      ]

      if len(__a) > 0:
         return completion[: min(__a)]
      else:
         return completion







'''simple docstring'''






from packaging import version

from .import_utils import is_accelerate_available


if is_accelerate_available():
      import accelerate




def        _SCREAMING_SNAKE_CASE  (				A  :       Optional[Any]  )    -> int:







   """simple docstring"""
   if not is_accelerate_available():
      return method
   __snake_case				:      Optional[Any]     							= version.parse(accelerate.__version__  ).base_version
   if version.parse(A  ) < version.parse('0.17.0'  ):
      return method

   def wrapper(self  :       Optional[Any]  ,		*A  :       Optional[Any]  ,		**A  :       Optional[int]  ):
      if hasattr(self  ,		'_hf_hook'  ) and hasattr(self._hf_hook  ,		'pre_forward'  ):
         self._hf_hook.pre_forward(self  )
      return method(self  ,		*A  ,		**A  )

   return wrapper