
'''simple docstring'''







import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_snake_case				:   Optional[Any]     =				logging.get_logger(__name__)

_snake_case				:   Optional[Any]     =				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_snake_case				:   int     =				{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

_snake_case				:   List[Any]     =				{
    """Salesforce/codegen-350M-mono""": 2_048,
}


class 			lowerCAmelCase (				__UpperCAmelCase			):
		a							:					Optional[int]       =					VOCAB_FILES_NAMES
		a							:					Dict       =					PRETRAINED_VOCAB_FILES_MAP
		a							:					Any       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a							:					List[Any]       =					["""input_ids""", """attention_mask"""]
		a							:					List[str]       =					CodeGenTokenizer




		def __init__(				self					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase="<|endoftext|>"					,						UpperCamelCase="<|endoftext|>"					,						UpperCamelCase="<|endoftext|>"					,						UpperCamelCase=False					,						**UpperCamelCase					,						):
								super().__init__(
								    UpperCamelCase					,						UpperCamelCase					,						tokenizer_file=UpperCamelCase					,						unk_token=UpperCamelCase					,						bos_token=UpperCamelCase					,						eos_token=UpperCamelCase					,						add_prefix_space=UpperCamelCase					,						**UpperCamelCase					,						)

								if kwargs.pop("add_bos_token"					,						UpperCamelCase      ):
														_SCREAMING_SNAKE_CASE				= kwargs.pop("name_or_path"					,						""      )
														raise ValueError(
														    "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
														    "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
														    F'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
														    F'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
														    "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
														    " so that the fast tokenizer works correctly."      )

								_SCREAMING_SNAKE_CASE				= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
								if pre_tok_state.get("add_prefix_space"					,						UpperCamelCase      ) != add_prefix_space:
														_SCREAMING_SNAKE_CASE				= getattr(UpperCamelCase					,						pre_tok_state.pop("type"      )      )
														_SCREAMING_SNAKE_CASE				= add_prefix_space
														_SCREAMING_SNAKE_CASE				= pre_tok_class(**UpperCamelCase      )

								_SCREAMING_SNAKE_CASE				= add_prefix_space




		def     lowercase       (				self					,						*UpperCamelCase					,						**UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= kwargs.get("is_split_into_words"					,						UpperCamelCase      )
								assert self.add_prefix_space or not is_split_into_words, (
								    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*UpperCamelCase					,						**UpperCamelCase      )




		def     lowercase       (				self					,						*UpperCamelCase					,						**UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= kwargs.get("is_split_into_words"					,						UpperCamelCase      )

								assert self.add_prefix_space or not is_split_into_words, (
								    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*UpperCamelCase					,						**UpperCamelCase      )




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase = None      ):
								_SCREAMING_SNAKE_CASE				= self._tokenizer.model.save(UpperCamelCase					,						name=UpperCamelCase      )
								return tuple(UpperCamelCase      )




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase = False					,						UpperCamelCase = None					,						UpperCamelCase = None					,						**UpperCamelCase					,						):
								_SCREAMING_SNAKE_CASE				= super().decode(
								    token_ids=UpperCamelCase					,						skip_special_tokens=UpperCamelCase					,						clean_up_tokenization_spaces=UpperCamelCase					,						**UpperCamelCase					,						)

								if truncate_before_pattern is not None and len(UpperCamelCase      ) > 0:
														_SCREAMING_SNAKE_CASE				= self.truncate(UpperCamelCase					,						UpperCamelCase      )

								return decoded_text




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase      ):
								def find_re(UpperCamelCase					,						UpperCamelCase					,						UpperCamelCase      ):
														_SCREAMING_SNAKE_CASE				= pattern.search(UpperCamelCase					,						UpperCamelCase      )
														return m.start() if m else -1

								_SCREAMING_SNAKE_CASE				= [re.compile(UpperCamelCase					,						re.MULTILINE      ) for pattern in truncate_before_pattern]

								_SCREAMING_SNAKE_CASE				= list(re.finditer("^print"					,						UpperCamelCase					,						re.MULTILINE      )      )

								if len(UpperCamelCase      ) > 1:
														_SCREAMING_SNAKE_CASE				= completion[: prints[1].start()]

								_SCREAMING_SNAKE_CASE				= list(re.finditer("^def"					,						UpperCamelCase					,						re.MULTILINE      )      )

								if len(UpperCamelCase      ) > 1:
														_SCREAMING_SNAKE_CASE				= completion[: defs[1].start()]

								_SCREAMING_SNAKE_CASE				= 0

								_SCREAMING_SNAKE_CASE				= [
								    pos for pos in [find_re(UpperCamelCase					,						UpperCamelCase					,						UpperCamelCase      ) for terminal in terminals] if pos != -1
								]

								if len(UpperCamelCase      ) > 0:
														return completion[: min(UpperCamelCase      )]
								else:
														return completion

'''simple docstring'''







import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler


class 			lowerCAmelCase :




		def __init__(				self					,						UpperCamelCase      ):
								if isinstance(UpperCamelCase					,						UpperCamelCase      ):
														# Don't modify user's data should they want to reuse it (e.g. in tests), because once we
														# modified it, it will not be accepted here again, since `auto` values would have been overridden
														_SCREAMING_SNAKE_CASE				= deepcopy(UpperCamelCase      )
								elif os.path.exists(UpperCamelCase      ):
														with io.open(UpperCamelCase					,						"r"					,						encoding="utf-8"      ) as f:
																				_SCREAMING_SNAKE_CASE				= json.load(UpperCamelCase      )
								else:
														try:
																				_SCREAMING_SNAKE_CASE				= baseaa.urlsafe_baadecode(UpperCamelCase      ).decode("utf-8"      )
																				_SCREAMING_SNAKE_CASE				= json.loads(UpperCamelCase      )
														except (UnicodeDecodeError, AttributeError, ValueError):
																				raise ValueError(
																				    F'Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}'      )

								_SCREAMING_SNAKE_CASE				= config

								self.set_stage_and_offload()




		def     lowercase       (				self      ):
								# zero stage - this is done as early as possible, before model is created, to allow
								# ``is_deepspeed_zero3_enabled`` query and getting to the early deepspeed config object
								# during ``zero.Init()`` which needs to know the dtype, and some other hparams.
								_SCREAMING_SNAKE_CASE				= self.get_value("zero_optimization.stage"					,						-1      )

								# offload
								_SCREAMING_SNAKE_CASE				= False
								if self.is_zeroa() or self.is_zeroa():
														_SCREAMING_SNAKE_CASE				= set(["cpu", "nvme"]      )
														_SCREAMING_SNAKE_CASE				= set(
														    [
														        self.get_value("zero_optimization.offload_optimizer.device"      ),
														        self.get_value("zero_optimization.offload_param.device"      ),
														    ]      )
														if len(offload_devices & offload_devices_valid      ) > 0:
																				_SCREAMING_SNAKE_CASE				= True




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= self.config

								# find the config node of interest if it exists
								_SCREAMING_SNAKE_CASE				= ds_key_long.split("."      )
								_SCREAMING_SNAKE_CASE				= nodes.pop()
								for node in nodes:
														_SCREAMING_SNAKE_CASE				= config.get(UpperCamelCase      )
														if config is None:
																				return None, ds_key

								return config, ds_key




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase=None      ):
								_SCREAMING_SNAKE_CASE,							_SCREAMING_SNAKE_CASE				= self.find_config_node(UpperCamelCase      )
								if config is None:
														return default
								return config.get(UpperCamelCase					,						UpperCamelCase      )




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase=False      ):
								_SCREAMING_SNAKE_CASE				= self.config

								# find the config node of interest if it exists
								_SCREAMING_SNAKE_CASE				= ds_key_long.split("."      )
								for node in nodes:
														_SCREAMING_SNAKE_CASE				= config
														_SCREAMING_SNAKE_CASE				= config.get(UpperCamelCase      )
														if config is None:
																				if must_exist:
																										raise ValueError(F'Can\'t find {ds_key_long} entry in the config: {self.config}'      )
																				else:
																										return

        # if found remove it
								if parent_config is not None:
														parent_config.pop(UpperCamelCase      )




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= self.get_value(UpperCamelCase      )
								return False if value is None else bool(UpperCamelCase      )




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= self.get_value(UpperCamelCase      )
								return False if value is None else not bool(UpperCamelCase      )




		def     lowercase       (				self      ):
								return self._stage == 2




		def     lowercase       (				self      ):
								return self._stage == 3




		def     lowercase       (				self      ):
								return self._offload


class 			lowerCAmelCase :




		def __init__(				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= engine




		def     lowercase       (				self					,						UpperCamelCase					,						**UpperCamelCase      ):
								# runs backpropagation and handles mixed precision
								self.engine.backward(UpperCamelCase					,						**UpperCamelCase      )

								# Deepspeed's `engine.step` performs the following operations:
								# - gradient accumulation check
								# - gradient clipping
								# - optimizer step
								# - zero grad
								# - checking overflow
								# - lr_scheduler step (only if engine.lr_scheduler is not None)
								self.engine.step()
								# and this plugin overrides the above calls with no-ops when Accelerate runs under
								# Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
								# training loop that works transparently under many training regimes.


class 			lowerCAmelCase (				__UpperCAmelCase			):




		def __init__(				self					,						UpperCamelCase      ):
								super().__init__(UpperCamelCase					,						device_placement=UpperCamelCase					,						scaler=UpperCamelCase      )
								_SCREAMING_SNAKE_CASE				= hasattr(self.optimizer					,						"overflow"      )




		def     lowercase       (				self					,						UpperCamelCase=None      ):
								pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed




		def     lowercase       (				self      ):
								pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed




		@property
		def     lowercase       (				self      ):
								if self.__has_overflow__:
														return self.optimizer.overflow
								return False


class 			lowerCAmelCase (				__UpperCAmelCase			):




		def __init__(				self					,						UpperCamelCase					,						UpperCamelCase      ):
								super().__init__(UpperCamelCase					,						UpperCamelCase      )




		def     lowercase       (				self      ):
								pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed


class 			lowerCAmelCase :




		def __init__(				self					,						UpperCamelCase					,						UpperCamelCase=0.0_01					,						UpperCamelCase=0					,						**UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= params
								_SCREAMING_SNAKE_CASE				= lr
								_SCREAMING_SNAKE_CASE				= weight_decay
								_SCREAMING_SNAKE_CASE				= kwargs


class 			lowerCAmelCase :




		def __init__(				self					,						UpperCamelCase					,						UpperCamelCase=None					,						UpperCamelCase=0					,						**UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= optimizer
								_SCREAMING_SNAKE_CASE				= total_num_steps
								_SCREAMING_SNAKE_CASE				= warmup_num_steps
								_SCREAMING_SNAKE_CASE				= kwargs