



import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(">=", FSDP_PYTORCH_VERSION):
			import torch.distributed.checkpoint as dist_cp
			from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
			from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
			from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
			from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


SCREAMING_SNAKE_CASE     				=						get_logger(__name__)







def snake_case__  (			__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE=0 )		->							int:
		os.makedirs(__SCREAMING_SNAKE_CASE			,     exist_ok=__SCREAMING_SNAKE_CASE )
		with FSDP.state_dict_type(
		    __SCREAMING_SNAKE_CASE			,     fsdp_plugin.state_dict_type			,     fsdp_plugin.state_dict_config			,     fsdp_plugin.optim_state_dict_config ):
				UpperCAmelCase_						=				model.state_dict()
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						UpperCAmelCase_						=				f'''{MODEL_NAME}.bin''' if model_index == 0 else f'''{MODEL_NAME}_{model_index}.bin'''
						UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
						if accelerator.process_index == 0:
								logger.info(f'''Saving model to {output_model_file}''' )
								torch.save(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
								logger.info(f'''Model saved to {output_model_file}''' )
				elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
						UpperCAmelCase_						=				(
						    f'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
						    if model_index == 0
						    else f'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
						)
						UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
						logger.info(f'''Saving model to {output_model_file}''' )
						torch.save(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
						logger.info(f'''Model saved to {output_model_file}''' )
				elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
						UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     f'''{MODEL_NAME}_{model_index}''' )
						os.makedirs(__SCREAMING_SNAKE_CASE			,     exist_ok=__SCREAMING_SNAKE_CASE )
						logger.info(f'''Saving model to {ckpt_dir}''' )
						UpperCAmelCase_						=				{"model": state_dict}

						dist_cp.save_state_dict(
						    state_dict=__SCREAMING_SNAKE_CASE			,     storage_writer=dist_cp.FileSystemWriter(__SCREAMING_SNAKE_CASE )			,     planner=DefaultSavePlanner()			,     )
						logger.info(f'''Model saved to {ckpt_dir}''' )







def snake_case__  (			__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE=0 )		->							List[Any]:
		accelerator.wait_for_everyone()
		with FSDP.state_dict_type(
		    __SCREAMING_SNAKE_CASE			,     fsdp_plugin.state_dict_type			,     fsdp_plugin.state_dict_config			,     fsdp_plugin.optim_state_dict_config ):
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						if type(__SCREAMING_SNAKE_CASE ) != FSDP and accelerator.process_index != 0:
								if not fsdp_plugin.sync_module_states:
										raise ValueError(
										    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
										    "initializing FSDP object" )
								return
						UpperCAmelCase_						=				f'''{MODEL_NAME}.bin''' if model_index == 0 else f'''{MODEL_NAME}_{model_index}.bin'''
						UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
						logger.info(f'''Loading model from {input_model_file}''' )
						UpperCAmelCase_						=				torch.load(__SCREAMING_SNAKE_CASE )
						logger.info(f'''Model loaded from {input_model_file}''' )
				elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
						UpperCAmelCase_						=				(
						    f'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
						    if model_index == 0
						    else f'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
						)
						UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
						logger.info(f'''Loading model from {input_model_file}''' )
						UpperCAmelCase_						=				torch.load(__SCREAMING_SNAKE_CASE )
						logger.info(f'''Model loaded from {input_model_file}''' )
				elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
						UpperCAmelCase_						=				(
						    os.path.join(__SCREAMING_SNAKE_CASE			,     f'''{MODEL_NAME}_{model_index}''' )
						    if f'''{MODEL_NAME}''' not in input_dir
						    else input_dir
						)
						logger.info(f'''Loading model from {ckpt_dir}''' )
						UpperCAmelCase_						=				{"model": model.state_dict()}
						dist_cp.load_state_dict(
						    state_dict=__SCREAMING_SNAKE_CASE			,     storage_reader=dist_cp.FileSystemReader(__SCREAMING_SNAKE_CASE )			,     planner=DefaultLoadPlanner()			,     )
						UpperCAmelCase_						=				state_dict["model"]
						logger.info(f'''Model loaded from {ckpt_dir}''' )
				model.load_state_dict(__SCREAMING_SNAKE_CASE )







def snake_case__  (			__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE=0 )		->							Optional[Any]:
		os.makedirs(__SCREAMING_SNAKE_CASE			,     exist_ok=__SCREAMING_SNAKE_CASE )
		with FSDP.state_dict_type(
		    __SCREAMING_SNAKE_CASE			,     fsdp_plugin.state_dict_type			,     fsdp_plugin.state_dict_config			,     fsdp_plugin.optim_state_dict_config ):
				UpperCAmelCase_						=				FSDP.optim_state_dict(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						if accelerator.process_index == 0:
								UpperCAmelCase_						=				(
								    f'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else f'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
								)
								UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
								logger.info(f'''Saving Optimizer state to {output_optimizer_file}''' )
								torch.save(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
								logger.info(f'''Optimizer state saved in {output_optimizer_file}''' )
				else:
						UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     f'''{OPTIMIZER_NAME}_{optimizer_index}''' )
						os.makedirs(__SCREAMING_SNAKE_CASE			,     exist_ok=__SCREAMING_SNAKE_CASE )
						logger.info(f'''Saving Optimizer state to {ckpt_dir}''' )
						dist_cp.save_state_dict(
						    state_dict={"optimizer": optim_state}			,     storage_writer=dist_cp.FileSystemWriter(__SCREAMING_SNAKE_CASE )			,     planner=DefaultSavePlanner()			,     )
						logger.info(f'''Optimizer state saved in {ckpt_dir}''' )







def snake_case__  (			__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE=0 )		->							Optional[Any]:
		accelerator.wait_for_everyone()
		with FSDP.state_dict_type(
		    __SCREAMING_SNAKE_CASE			,     fsdp_plugin.state_dict_type			,     fsdp_plugin.state_dict_config			,     fsdp_plugin.optim_state_dict_config ):
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						UpperCAmelCase_						=				None
						# below check should work but currently it isn't working (mostly opytorch issue),
						# in the meantime disabling it at the cost of excess memory usage
						# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
						UpperCAmelCase_						=				(
						    f'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else f'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
						)
						UpperCAmelCase_						=				os.path.join(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
						logger.info(f'''Loading Optimizer state from {input_optimizer_file}''' )
						UpperCAmelCase_						=				torch.load(__SCREAMING_SNAKE_CASE )
						logger.info(f'''Optimizer state loaded from {input_optimizer_file}''' )
				else:
						UpperCAmelCase_						=				(
						    os.path.join(__SCREAMING_SNAKE_CASE			,     f'''{OPTIMIZER_NAME}_{optimizer_index}''' )
						    if f'''{OPTIMIZER_NAME}''' not in input_dir
						    else input_dir
						)
						logger.info(f'''Loading Optimizer from {ckpt_dir}''' )
						UpperCAmelCase_						=				load_sharded_optimizer_state_dict(
						    model_state_dict=model.state_dict()			,     optimizer_key="optimizer"			,     storage_reader=dist_cp.FileSystemReader(__SCREAMING_SNAKE_CASE )			,     )
						UpperCAmelCase_						=				optim_state["optimizer"]
						logger.info(f'''Optimizer loaded from {ckpt_dir}''' )
				UpperCAmelCase_						=				FSDP.optim_state_dict_to_load(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE )
				optimizer.load_state_dict(__SCREAMING_SNAKE_CASE )










import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


SCREAMING_SNAKE_CASE     				=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE     				=						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE     				=						{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE     				=						{
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


SCREAMING_SNAKE_CASE     				=						{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}



class 	lowerCamelCase      (						lowercase__  ):







					'''simple docstring'''







					lowerCAmelCase_				:    Any									=  VOCAB_FILES_NAMES
					lowerCAmelCase_				:    List[str]									=  PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_				:    str									=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_				:    Union[str, Any]									=  PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase_				:    int									=  ['input_ids', 'attention_mask']
					lowerCAmelCase_				:    str									=  DistilBertTokenizer



					def __init__(			self    ,	lowerCAmelCase=None    ,	lowerCAmelCase=None    ,	lowerCAmelCase=True    ,	lowerCAmelCase="[UNK]"    ,	lowerCAmelCase="[SEP]"    ,	lowerCAmelCase="[PAD]"    ,	lowerCAmelCase="[CLS]"    ,	lowerCAmelCase="[MASK]"    ,	lowerCAmelCase=True    ,	lowerCAmelCase=None    ,	**lowerCAmelCase    ,	):
							super().__init__(
							    lowerCAmelCase    ,	tokenizer_file=lowerCAmelCase    ,	do_lower_case=lowerCAmelCase    ,	unk_token=lowerCAmelCase    ,	sep_token=lowerCAmelCase    ,	pad_token=lowerCAmelCase    ,	cls_token=lowerCAmelCase    ,	mask_token=lowerCAmelCase    ,	tokenize_chinese_chars=lowerCAmelCase    ,	strip_accents=lowerCAmelCase    ,	**lowerCAmelCase    ,	)

							UpperCAmelCase_						=				json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
							if (
							    normalizer_state.get("lowercase"    ,	lowerCAmelCase     ) != do_lower_case
							    or normalizer_state.get("strip_accents"    ,	lowerCAmelCase     ) != strip_accents
							    or normalizer_state.get("handle_chinese_chars"    ,	lowerCAmelCase     ) != tokenize_chinese_chars
							):
									UpperCAmelCase_						=				getattr(lowerCAmelCase    ,	normalizer_state.pop("type"     )     )
									UpperCAmelCase_						=				do_lower_case
									UpperCAmelCase_						=				strip_accents
									UpperCAmelCase_						=				tokenize_chinese_chars
									UpperCAmelCase_						=				normalizer_class(**lowerCAmelCase     )

							UpperCAmelCase_						=				do_lower_case



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase=None     ):
							UpperCAmelCase_						=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

							return output



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase = None     ):
							UpperCAmelCase_						=				[self.sep_token_id]
							UpperCAmelCase_						=				[self.cls_token_id]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep     ) * [0]
							return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase = None     ):
							UpperCAmelCase_						=				self._tokenizer.model.save(lowerCAmelCase    ,	name=lowerCAmelCase     )
							return tuple(lowerCAmelCase     )

