






"""simple docstring"""





import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
    if is_torch_available():
        import torch
    if is_tf_available():
        import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCamelCase_	:				Dict										= logging.get_logger(__name__)

UpperCamelCase_	:				str										= {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase_	:				int										= {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase_	:				str										= {
    """Salesforce/codegen-350M-mono""": 2048,
}


class      _lowercase (     lowerCAmelCase						):
      _a   :     Optional[Any]								=   VOCAB_FILES_NAMES
      _a   :     str								=   PRETRAINED_VOCAB_FILES_MAP
      _a   :     Optional[Any]								=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _a   :     Any								=   ['''input_ids''', '''attention_mask''']
      _a   :     int								=   CodeGenTokenizer


      def __init__(    self				:	int							,   a				:	Dict=None							,   a				:	Dict=None							,   a				:	Optional[int]=None							,   a				:	Union[str, Any]="<|endoftext|>"							,   a				:	str="<|endoftext|>"							,   a				:	List[str]="<|endoftext|>"							,   a				:	Any=False							,   **a				:	str							,   ):





            """simple docstring"""







            super().__init__(
                a							,   a							,   tokenizer_file=a							,   unk_token=a							,   bos_token=a							,   eos_token=a							,   add_prefix_space=a							,   **a							,   )

            if kwargs.pop('''add_bos_token'''							,   a						):
                  __snake_case      :    Dict  			=kwargs.pop('''name_or_path'''							,   ''''''						)
                  raise ValueError(
                      '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
                      '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
                      f'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
                      f'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
                      '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
                      ''' so that the fast tokenizer works correctly.'''						)

            __snake_case      :    Optional[Any]  			=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
            if pre_tok_state.get('''add_prefix_space'''							,   a						) != add_prefix_space:
                  __snake_case      :    List[str]  			=getattr(a							,   pre_tok_state.pop('''type'''						)						)
                  __snake_case      :    List[Any]  			=add_prefix_space
                  __snake_case      :    List[str]  			=pre_tok_class(**a						)

            __snake_case      :    Optional[int]  			=add_prefix_space


      def _UpperCamelCase						(    self				:	int							,   *a				:	Dict							,   **a				:	Optional[Any]						):





            """simple docstring"""







            __snake_case      :    Tuple  			=kwargs.get('''is_split_into_words'''							,   a						)
            assert self.add_prefix_space or not is_split_into_words, (
                f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._batch_encode_plus(*a							,   **a						)


      def _UpperCamelCase						(    self				:	List[str]							,   *a				:	Any							,   **a				:	int						):





            """simple docstring"""







            __snake_case      :    Any  			=kwargs.get('''is_split_into_words'''							,   a						)

            assert self.add_prefix_space or not is_split_into_words, (
                f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._encode_plus(*a							,   **a						)


      def _UpperCamelCase						(    self				:	Optional[Any]							,   a				:	str							,   a				:	Optional[str] = None						):





            """simple docstring"""







            __snake_case      :    Any  			=self._tokenizer.model.save(a							,   name=a						)
            return tuple(a						)


      def _UpperCamelCase						(    self				:	Optional[int]							,   a				:	Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]							,   a				:	bool = False							,   a				:	bool = None							,   a				:	Optional[List[str]] = None							,   **a				:	Optional[int]							,   ):





            """simple docstring"""







            __snake_case      :    Any  			=super().decode(
                token_ids=a							,   skip_special_tokens=a							,   clean_up_tokenization_spaces=a							,   **a							,   )

            if truncate_before_pattern is not None and len(a						) > 0:
                  __snake_case      :    List[Any]  			=self.truncate(a							,   a						)

            return decoded_text







      def _UpperCamelCase						(    self				:	Optional[Any]							,   a				:	Optional[Any]							,   a				:	Dict						):





            """simple docstring"""







            def find_re(a				:	Optional[Any]							,   a				:	Dict							,   a				:	int						):
                  __snake_case      :    str  			=pattern.search(a							,   a						)
                  return m.start() if m else -1

            __snake_case      :    Tuple  			=[re.compile(a							,   re.MULTILINE						) for pattern in truncate_before_pattern]

            __snake_case      :    List[str]  			=list(re.finditer('''^print'''							,   a							,   re.MULTILINE						)						)

            if len(a						) > 1:
                  __snake_case      :    str  			=completion[: prints[1].start()]

            __snake_case      :    Any  			=list(re.finditer('''^def'''							,   a							,   re.MULTILINE						)						)

            if len(a						) > 1:
                  __snake_case      :    Any  			=completion[: defs[1].start()]

            __snake_case      :    int  			=0

            __snake_case      :    Optional[int]  			=[
                pos for pos in [find_re(a							,   a							,   a						) for terminal in terminals] if pos != -1
            ]

            if len(a						) > 0:
                  return completion[: min(a						)]
            else:
                  return completion










"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import _LazyModule


UpperCamelCase_	:				Union[str, Any]										= {"""tokenization_wav2vec2_phoneme""": ["""Wav2Vec2PhonemeCTCTokenizer"""]}


if TYPE_CHECKING:
    from .tokenization_wavaveca_phoneme import WavaVecaPhonemeCTCTokenizer
else:
    import sys

    UpperCamelCase_	:				List[Any]										= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


