


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A							:					Optional[int]     						=     logging.get_logger(__name__)
A							:					List[Any]     						=     {'vocab_file': 'spiece.model'}

A							:					int     						=     {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

A							:					Optional[Any]     						=     {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

A							:					List[Any]     						=     '▁'






class     UpperCamelCase(  _a   ):
					snake_case_		:				Union[str, Any]										=	VOCAB_FILES_NAMES
					snake_case_		:				Dict										=	PRETRAINED_VOCAB_FILES_MAP
					snake_case_		:				Any										=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


					def __init__(						self	:		Tuple     ,    SCREAMING_SNAKE_CASE	:		Optional[int]     ,    SCREAMING_SNAKE_CASE	:		Tuple=True     ,    SCREAMING_SNAKE_CASE	:		List[Any]=True     ,    SCREAMING_SNAKE_CASE	:		List[str]=False     ,    SCREAMING_SNAKE_CASE	:		int="[CLS]"     ,    SCREAMING_SNAKE_CASE	:		int="[SEP]"     ,    SCREAMING_SNAKE_CASE	:		Union[str, Any]="<unk>"     ,    SCREAMING_SNAKE_CASE	:		Any="[SEP]"     ,    SCREAMING_SNAKE_CASE	:		List[Any]="<pad>"     ,    SCREAMING_SNAKE_CASE	:		Dict="[CLS]"     ,    SCREAMING_SNAKE_CASE	:		List[Any]="[MASK]"     ,    SCREAMING_SNAKE_CASE	:		Optional[Dict[str, Any]] = None     ,    **SCREAMING_SNAKE_CASE	:		Tuple     ,    )			->	None:




											'''simple docstring'''

											__snake_case				=		(
											    AddedToken(SCREAMING_SNAKE_CASE     ,    lstrip=SCREAMING_SNAKE_CASE     ,    rstrip=SCREAMING_SNAKE_CASE     ,    normalized=SCREAMING_SNAKE_CASE						)
											    if isinstance(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)
											    else mask_token
											)

											__snake_case				=		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=SCREAMING_SNAKE_CASE     ,    remove_space=SCREAMING_SNAKE_CASE     ,    keep_accents=SCREAMING_SNAKE_CASE     ,    bos_token=SCREAMING_SNAKE_CASE     ,    eos_token=SCREAMING_SNAKE_CASE     ,    unk_token=SCREAMING_SNAKE_CASE     ,    sep_token=SCREAMING_SNAKE_CASE     ,    pad_token=SCREAMING_SNAKE_CASE     ,    cls_token=SCREAMING_SNAKE_CASE     ,    mask_token=SCREAMING_SNAKE_CASE     ,    sp_model_kwargs=self.sp_model_kwargs     ,    **SCREAMING_SNAKE_CASE     ,    )

											__snake_case				=		do_lower_case
											__snake_case				=		remove_space
											__snake_case				=		keep_accents
											__snake_case				=		vocab_file

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(SCREAMING_SNAKE_CASE						)


					@property
					def 							SCREAMING_SNAKE_CASE_  (						self	:		List[str]						)			->	Optional[Any]:




											'''simple docstring'''

											return len(self.sp_model						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Any						)			->	Dict:




											'''simple docstring'''

											__snake_case				=		{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE						): i for i in range(self.vocab_size						)}
											vocab.update(self.added_tokens_encoder						)
											return vocab


					def __getstate__(						self	:		Dict						)			->	int:




											'''simple docstring'''

											__snake_case				=		self.__dict__.copy()
											__snake_case				=		None
											return state


					def __setstate__(						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		Dict						)			->	Any:




											'''simple docstring'''

											__snake_case				=		d

											# for backward compatibility
											if not hasattr(self     ,    "sp_model_kwargs"						):
																	__snake_case				=		{}

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(self.vocab_file						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple     ,    SCREAMING_SNAKE_CASE	:		Optional[int]						)			->	Any:




											'''simple docstring'''

											if self.remove_space:
																	__snake_case				=		" ".join(inputs.strip().split()						)
											else:
																	__snake_case				=		inputs
											__snake_case				=		outputs.replace("``"     ,    "\""						).replace("''"     ,    "\""						)

											if not self.keep_accents:
																	__snake_case				=		unicodedata.normalize("NFKD"     ,    SCREAMING_SNAKE_CASE						)
																	__snake_case				=		"".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE						)]						)
											if self.do_lower_case:
																	__snake_case				=		outputs.lower()

											return outputs


					def 							SCREAMING_SNAKE_CASE_  (						self	:		int     ,    SCREAMING_SNAKE_CASE	:		str						)			->	List[str]:




											'''simple docstring'''

											__snake_case				=		self.preprocess_text(SCREAMING_SNAKE_CASE						)
											__snake_case				=		self.sp_model.encode(SCREAMING_SNAKE_CASE     ,    out_type=SCREAMING_SNAKE_CASE						)
											__snake_case				=		[]
											for piece in pieces:
																	if len(SCREAMING_SNAKE_CASE						) > 1 and piece[-1] == str(","						) and piece[-2].isdigit():
																							__snake_case				=		self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE     ,    ""						)						)
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																													if len(cur_pieces[0]						) == 1:
																																			__snake_case				=		cur_pieces[1:]
																													else:
																																			__snake_case				=		cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]						)
																							new_pieces.extend(SCREAMING_SNAKE_CASE						)
																	else:
																							new_pieces.append(SCREAMING_SNAKE_CASE						)

											return new_pieces


					def 							SCREAMING_SNAKE_CASE_  (						self	:		str     ,    SCREAMING_SNAKE_CASE	:		Optional[Any]						)			->	int:




											'''simple docstring'''

											return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Union[str, Any]     ,    SCREAMING_SNAKE_CASE	:		Tuple						)			->	Optional[Any]:




											'''simple docstring'''

											return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		int     ,    SCREAMING_SNAKE_CASE	:		Union[str, Any]						)			->	Dict:




											'''simple docstring'''

											__snake_case				=		[]
											__snake_case				=		""
											__snake_case				=		False
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							if not prev_is_special:
																													out_string += " "
																							out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE						) + token
																							__snake_case				=		True
																							__snake_case				=		[]
																	else:
																							current_sub_tokens.append(SCREAMING_SNAKE_CASE						)
																							__snake_case				=		False
											out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE						)
											return out_string.strip()


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict     ,    SCREAMING_SNAKE_CASE	:		List[int]     ,    SCREAMING_SNAKE_CASE	:		Optional[List[int]] = None						)			->	List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[self.cls_token_id]
											if token_ids_a is None:
																	return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict     ,    SCREAMING_SNAKE_CASE	:		List[int]     ,    SCREAMING_SNAKE_CASE	:		Optional[List[int]] = None     ,    SCREAMING_SNAKE_CASE	:		bool = False						)			->	List[int]:




											'''simple docstring'''

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=SCREAMING_SNAKE_CASE     ,    token_ids_a=SCREAMING_SNAKE_CASE     ,    already_has_special_tokens=SCREAMING_SNAKE_CASE						)

											if token_ids_a is not None:
																	return [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1]
											return [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1]


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		List[int]     ,    SCREAMING_SNAKE_CASE	:		Optional[List[int]] = None						)			->	List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		str     ,    SCREAMING_SNAKE_CASE	:		Optional[str] = None						)			->	Tuple[str]:




											'''simple docstring'''

											if not os.path.isdir(SCREAMING_SNAKE_CASE						):
																	logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
																	return
											__snake_case				=		os.path.join(
											    SCREAMING_SNAKE_CASE     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

											if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE						) and os.path.isfile(self.vocab_file						):
																	copyfile(self.vocab_file     ,    SCREAMING_SNAKE_CASE						)
											elif not os.path.isfile(self.vocab_file						):
																	with open(SCREAMING_SNAKE_CASE     ,    "wb"						) as fi:
																							__snake_case				=		self.sp_model.serialized_model_proto()
																							fi.write(SCREAMING_SNAKE_CASE						)

											return (out_vocab_file,)









import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler






class     UpperCamelCase:


					def __init__(						self	:		List[str]     ,    SCREAMING_SNAKE_CASE	:		Dict						)			->	str:




											'''simple docstring'''

											if isinstance(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						):
																	# Don't modify user's data should they want to reuse it (e.g. in tests), because once we
																	# modified it, it will not be accepted here again, since `auto` values would have been overridden
																	__snake_case				=		deepcopy(SCREAMING_SNAKE_CASE						)
											elif os.path.exists(SCREAMING_SNAKE_CASE						):
																	with io.open(SCREAMING_SNAKE_CASE     ,    "r"     ,    encoding="utf-8"						) as f:
																							__snake_case				=		json.load(SCREAMING_SNAKE_CASE						)
											else:
																	try:
																							__snake_case				=		baseaa.urlsafe_baadecode(SCREAMING_SNAKE_CASE						).decode("utf-8"						)
																							__snake_case				=		json.loads(SCREAMING_SNAKE_CASE						)
																	except (UnicodeDecodeError, AttributeError, ValueError):
																							raise ValueError(
																							    f'''Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}'''						)

											__snake_case				=		config

											self.set_stage_and_offload()


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Any						)			->	List[str]:




											'''simple docstring'''

											__snake_case				=		self.get_value("zero_optimization.stage"     ,    -1						)

											# offload
											__snake_case				=		False
											if self.is_zeroa() or self.is_zeroa():
																	__snake_case				=		set(["cpu", "nvme"]						)
																	__snake_case				=		set(
																	    [
																	        self.get_value("zero_optimization.offload_optimizer.device"						),
																	        self.get_value("zero_optimization.offload_param.device"						),
																	    ]						)
																	if len(offload_devices & offload_devices_valid						) > 0:
																							__snake_case				=		True


					def 							SCREAMING_SNAKE_CASE_  (						self	:		int     ,    SCREAMING_SNAKE_CASE	:		List[Any]						)			->	str:




											'''simple docstring'''

											__snake_case				=		self.config

											# find the config node of interest if it exists
											__snake_case				=		ds_key_long.split("."						)
											__snake_case				=		nodes.pop()
											for node in nodes:
																	__snake_case				=		config.get(SCREAMING_SNAKE_CASE						)
																	if config is None:
																							return None, ds_key

											return config, ds_key


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		str     ,    SCREAMING_SNAKE_CASE	:		Tuple=None						)			->	List[str]:




											'''simple docstring'''

											__snake_case		,					__snake_case				=		self.find_config_node(SCREAMING_SNAKE_CASE						)
											if config is None:
																	return default
											return config.get(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Any     ,    SCREAMING_SNAKE_CASE	:		Optional[int]     ,    SCREAMING_SNAKE_CASE	:		Tuple=False						)			->	Union[str, Any]:




											'''simple docstring'''

											__snake_case				=		self.config

											# find the config node of interest if it exists
											__snake_case				=		ds_key_long.split("."						)
											for node in nodes:
																	__snake_case				=		config
																	__snake_case				=		config.get(SCREAMING_SNAKE_CASE						)
																	if config is None:
																							if must_exist:
																													raise ValueError(f'''Can\'t find {ds_key_long} entry in the config: {self.config}'''						)
																							else:
																													return

        # if found remove it
											if parent_config is not None:
																	parent_config.pop(SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple     ,    SCREAMING_SNAKE_CASE	:		Any						)			->	Optional[Any]:




											'''simple docstring'''

											__snake_case				=		self.get_value(SCREAMING_SNAKE_CASE						)
											return False if value is None else bool(SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Any     ,    SCREAMING_SNAKE_CASE	:		Union[str, Any]						)			->	str:




											'''simple docstring'''

											__snake_case				=		self.get_value(SCREAMING_SNAKE_CASE						)
											return False if value is None else not bool(SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict						)			->	int:




											'''simple docstring'''

											return self._stage == 2


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[int]						)			->	List[str]:




											'''simple docstring'''

											return self._stage == 3






					def 							SCREAMING_SNAKE_CASE_  (						self	:		List[Any]						)			->	Optional[Any]:




											'''simple docstring'''

											return self._offload






class     UpperCamelCase:


					def __init__(						self	:		Union[str, Any]     ,    SCREAMING_SNAKE_CASE	:		Any						)			->	Optional[Any]:




											'''simple docstring'''

											__snake_case				=		engine






					def 							SCREAMING_SNAKE_CASE_  (						self	:		int     ,    SCREAMING_SNAKE_CASE	:		Tuple     ,    **SCREAMING_SNAKE_CASE	:		str						)			->	List[str]:




											'''simple docstring'''

											self.engine.backward(SCREAMING_SNAKE_CASE     ,    **SCREAMING_SNAKE_CASE						)

											# Deepspeed's `engine.step` performs the following operations:
											# - gradient accumulation check
											# - gradient clipping
											# - optimizer step
											# - zero grad
											# - checking overflow
											# - lr_scheduler step (only if engine.lr_scheduler is not None)
											self.engine.step()
											# and this plugin overrides the above calls with no-ops when Accelerate runs under
											# Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
											# training loop that works transparently under many training regimes.






class     UpperCamelCase(  _a   ):


					def __init__(						self	:		List[Any]     ,    SCREAMING_SNAKE_CASE	:		Dict						)			->	str:




											'''simple docstring'''

											super().__init__(SCREAMING_SNAKE_CASE     ,    device_placement=SCREAMING_SNAKE_CASE     ,    scaler=SCREAMING_SNAKE_CASE						)
											__snake_case				=		hasattr(self.optimizer     ,    "overflow"						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		List[Any]=None						)			->	str:




											'''simple docstring'''

											pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict						)			->	Any:




											'''simple docstring'''

											pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






					@property
					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[int]						)			->	Any:




											'''simple docstring'''

											if self.__has_overflow__:
																	return self.optimizer.overflow
											return False






class     UpperCamelCase(  _a   ):


					def __init__(						self	:		List[Any]     ,    SCREAMING_SNAKE_CASE	:		Any     ,    SCREAMING_SNAKE_CASE	:		str						)			->	Union[str, Any]:




											'''simple docstring'''

											super().__init__(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)






					def 							SCREAMING_SNAKE_CASE_  (						self	:		List[str]						)			->	Any:




											'''simple docstring'''

											pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






class     UpperCamelCase:


					def __init__(						self	:		Union[str, Any]     ,    SCREAMING_SNAKE_CASE	:		List[Any]     ,    SCREAMING_SNAKE_CASE	:		Optional[Any]=0.001     ,    SCREAMING_SNAKE_CASE	:		Tuple=0     ,    **SCREAMING_SNAKE_CASE	:		Optional[int]						)			->	List[str]:




											'''simple docstring'''

											__snake_case				=		params
											__snake_case				=		lr
											__snake_case				=		weight_decay
											__snake_case				=		kwargs






class     UpperCamelCase:


					def __init__(						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		List[Any]     ,    SCREAMING_SNAKE_CASE	:		Dict=None     ,    SCREAMING_SNAKE_CASE	:		Tuple=0     ,    **SCREAMING_SNAKE_CASE	:		Tuple						)			->	Tuple:




											'''simple docstring'''

											__snake_case				=		optimizer
											__snake_case				=		total_num_steps
											__snake_case				=		warmup_num_steps
											__snake_case				=		kwargs





