





import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
			if is_torch_available():
						import torch
			if is_tf_available():
						import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_a						=   logging.get_logger(__name__)

_a						=   {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_a						=   {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

_a						=   {
    "Salesforce/codegen-350M-mono": 2_048,
}






class       __A					(					lowerCAmelCase				):

			'''simple docstring'''







			lowerCAmelCase_							  =		VOCAB_FILES_NAMES
			lowerCAmelCase_							  =		PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_							  =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_							  =		["""input_ids""", """attention_mask"""]
			lowerCAmelCase_							  =		CodeGenTokenizer





			def __init__(					self ,						__lowerCAmelCase=None ,						__lowerCAmelCase=None ,						__lowerCAmelCase=None ,						__lowerCAmelCase="<|endoftext|>" ,						__lowerCAmelCase="<|endoftext|>" ,						__lowerCAmelCase="<|endoftext|>" ,						__lowerCAmelCase=False ,						**__lowerCAmelCase ,						):



				'''simple docstring'''







				super().__init__(
				    __lowerCAmelCase ,						__lowerCAmelCase ,						tokenizer_file=__lowerCAmelCase ,						unk_token=__lowerCAmelCase ,						bos_token=__lowerCAmelCase ,						eos_token=__lowerCAmelCase ,						add_prefix_space=__lowerCAmelCase ,						**__lowerCAmelCase ,						)

				if kwargs.pop('''add_bos_token''' ,						__lowerCAmelCase							):
					lowerCamelCase__              =	kwargs.pop('''name_or_path''' ,						''''''							)
					raise ValueError(
					    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
					    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
					    F'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
					    F'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
					    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
					    ''' so that the fast tokenizer works correctly.'''							)

				lowerCamelCase__              =	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
				if pre_tok_state.get('''add_prefix_space''' ,						__lowerCAmelCase							) != add_prefix_space:
					lowerCamelCase__              =	getattr(__lowerCAmelCase ,						pre_tok_state.pop('''type'''							)							)
					lowerCamelCase__              =	add_prefix_space
					lowerCamelCase__              =	pre_tok_class(**__lowerCAmelCase							)

				lowerCamelCase__              =	add_prefix_space





			def 	__lowerCamelCase       (					self ,						*__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	kwargs.get('''is_split_into_words''' ,						__lowerCAmelCase							)
				assert self.add_prefix_space or not is_split_into_words, (
				    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
				    "to use it with pretokenized inputs."
				)

				return super()._batch_encode_plus(*__lowerCAmelCase ,						**__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						*__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	kwargs.get('''is_split_into_words''' ,						__lowerCAmelCase							)

				assert self.add_prefix_space or not is_split_into_words, (
				    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
				    "to use it with pretokenized inputs."
				)

				return super()._encode_plus(*__lowerCAmelCase ,						**__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase = None							):



				'''simple docstring'''







				lowerCamelCase__              =	self._tokenizer.model.save(__lowerCAmelCase ,						name=__lowerCAmelCase							)
				return tuple(__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase = False ,						__lowerCAmelCase = None ,						__lowerCAmelCase = None ,						**__lowerCAmelCase ,						):



				'''simple docstring'''







				lowerCamelCase__              =	super().decode(
				    token_ids=__lowerCAmelCase ,						skip_special_tokens=__lowerCAmelCase ,						clean_up_tokenization_spaces=__lowerCAmelCase ,						**__lowerCAmelCase ,						)

				if truncate_before_pattern is not None and len(__lowerCAmelCase							) > 0:
					lowerCamelCase__              =	self.truncate(__lowerCAmelCase ,						__lowerCAmelCase							)

				return decoded_text





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase							):



				'''simple docstring'''







				def find_re(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							):
					lowerCamelCase__              =	pattern.search(__lowerCAmelCase ,						__lowerCAmelCase							)
					return m.start() if m else -1

				lowerCamelCase__              =	[re.compile(__lowerCAmelCase ,						re.MULTILINE							) for pattern in truncate_before_pattern]

				lowerCamelCase__              =	list(re.finditer('''^print''' ,						__lowerCAmelCase ,						re.MULTILINE							)							)

				if len(__lowerCAmelCase							) > 1:
					lowerCamelCase__              =	completion[: prints[1].start()]

				lowerCamelCase__              =	list(re.finditer('''^def''' ,						__lowerCAmelCase ,						re.MULTILINE							)							)

				if len(__lowerCAmelCase							) > 1:
					lowerCamelCase__              =	completion[: defs[1].start()]

				lowerCamelCase__              =	0

				lowerCamelCase__              =	[
				    pos for pos in [find_re(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							) for terminal in terminals] if pos != -1
				]

				if len(__lowerCAmelCase							) > 0:
					return completion[: min(__lowerCAmelCase							)]
				else:
					return completion










# This is the module that test_patching.py uses to test patch_submodule()

import os  # noqa: this is just for tests
import os as renamed_os  # noqa: this is just for tests
from os import path  # noqa: this is just for tests
from os import path as renamed_path  # noqa: this is just for tests
from os.path import join  # noqa: this is just for tests
from os.path import join as renamed_join  # noqa: this is just for tests


_a						=   open  # noqa: we just need to have a builtin inside this module to test it properly



