



'''simple docstring'''



from argparse import ArgumentParser

from . import BaseTransformersCLICommand



def   _A       (   _lowerCAmelCase					):



							"""simple docstring"""



							return DownloadCommand(args.model  ,   args.cache_dir  ,   args.force  ,   args.trust_remote_code					)





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				@staticmethod
				def 				__lowerCamelCase     (							_lowerCAmelCase  :  ArgumentParser):







											'''simple docstring'''

											__lowercase       	=parser.add_parser('download')
											download_parser.add_argument(
											    '--cache-dir'  ,       type=_lowerCAmelCase  ,       default=_lowerCAmelCase  ,       help='Path to location to store the models')
											download_parser.add_argument(
											    '--force'  ,       action='store_true'  ,       help='Force the model to be download even if already in cache-dir')
											download_parser.add_argument(
											    '--trust-remote-code'  ,       action='store_true'  ,       help='Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you\'ve reviewed the code as it will execute on your local machine'  ,       )
											download_parser.add_argument('model'  ,       type=_lowerCAmelCase  ,       help='Name of the model to download')
											download_parser.set_defaults(func=_lowerCAmelCase)

				def __init__(							self  :  Dict  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  bool  ,       _lowerCAmelCase  :  bool):







											'''simple docstring'''

											__lowercase       	=model
											__lowercase       	=cache
											__lowercase       	=force
											__lowercase       	=trust_remote_code

				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											from ..models.auto import AutoModel, AutoTokenizer

											AutoModel.from_pretrained(
											    self._model  ,       cache_dir=self._cache  ,       force_download=self._force  ,       trust_remote_code=self._trust_remote_code)
											AutoTokenizer.from_pretrained(
											    self._model  ,       cache_dir=self._cache  ,       force_download=self._force  ,       trust_remote_code=self._trust_remote_code)








'''simple docstring'''



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)

lowerCamelCase	 =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase	 =       {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase	 =       {
    """Salesforce/codegen-350M-mono""": 2048,
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    ["""input_ids""", """attention_mask"""]
				lowerCAmelCase__					    =    CodeGenTokenizer

				def __init__(							self  :  int  ,       _lowerCAmelCase  :  str=None  ,       _lowerCAmelCase  :  List[Any]=None  ,       _lowerCAmelCase  :  Optional[Any]=None  ,       _lowerCAmelCase  :  List[str]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[Any]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]=False  ,       **_lowerCAmelCase  :  Union[str, Any]  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       bos_token=_lowerCAmelCase  ,       eos_token=_lowerCAmelCase  ,       add_prefix_space=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if kwargs.pop('add_bos_token'  ,       _lowerCAmelCase):
																		__lowercase       	=kwargs.pop('name_or_path'  ,       '')
																		raise ValueError(
																		    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
																		    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
																		    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
																		    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
																		    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
																		    ' so that the fast tokenizer works correctly.')

											__lowercase       	=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																		__lowercase       	=getattr(_lowerCAmelCase  ,       pre_tok_state.pop('type'))
																		__lowercase       	=add_prefix_space
																		__lowercase       	=pre_tok_class(**_lowerCAmelCase)

											__lowercase       	=add_prefix_space

				def 				__lowerCamelCase     (							self  :  Any  ,       *_lowerCAmelCase  :  Tuple  ,       **_lowerCAmelCase  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)
											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[str]  ,       *_lowerCAmelCase  :  List[Any]  ,       **_lowerCAmelCase  :  List[str]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]  ,       _lowerCAmelCase  :  bool = False  ,       _lowerCAmelCase  :  bool = None  ,       _lowerCAmelCase  :  Optional[List[str]] = None  ,       **_lowerCAmelCase  :  Any  ,       ):







											'''simple docstring'''

											__lowercase       	=super().decode(
											    token_ids=_lowerCAmelCase  ,       skip_special_tokens=_lowerCAmelCase  ,       clean_up_tokenization_spaces=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if truncate_before_pattern is not None and len(_lowerCAmelCase) > 0:
																		__lowercase       	=self.truncate(_lowerCAmelCase  ,       _lowerCAmelCase)

											return decoded_text

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											def find_re(_lowerCAmelCase  :  List[Any]  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  str):
																		__lowercase       	=pattern.search(_lowerCAmelCase  ,       _lowerCAmelCase)
																		return m.start() if m else -1

											__lowercase       	=[re.compile(_lowerCAmelCase  ,       re.MULTILINE) for pattern in truncate_before_pattern]

											__lowercase       	=list(re.finditer('^print'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: prints[1].start()]

											__lowercase       	=list(re.finditer('^def'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: defs[1].start()]

											__lowercase       	=0

											__lowercase       	=[
											    pos for pos in [find_re(_lowerCAmelCase  ,       _lowerCAmelCase  ,       _lowerCAmelCase) for terminal in terminals] if pos != -1
											]

											if len(_lowerCAmelCase) > 0:
																		return completion[: min(_lowerCAmelCase)]
											else:
																		return completion



