






"""simple docstring"""
import itertools
import json
import linecache
import os
import pickle
import re
import socket
import string
from collections import Counter
from logging import getLogger
from pathlib import Path
from typing import Callable, Dict, Iterable, List

import git
import torch
from torch.utils.data import Dataset

from transformers import BartTokenizer, RagTokenizer, TaTokenizer






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=True   ,   snake_case__="pt"					):
	A_		:						List[Any]	    =   {'add_prefix_space': True} if isinstance(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					) and not line.startswith(""" """					) else {}
	A_		:						str	    =   padding_side
	return tokenizer(
	    [line]   ,   max_length=SCREAMING_SNAKE_CASE_   ,   padding="""max_length""" if pad_to_max_length else None   ,   truncation=SCREAMING_SNAKE_CASE_   ,   return_tensors=SCREAMING_SNAKE_CASE_   ,   add_special_tokens=SCREAMING_SNAKE_CASE_   ,   **SCREAMING_SNAKE_CASE_   ,   )






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__=None   ,   ):
	A_		:						Any	    =   input_ids.ne(SCREAMING_SNAKE_CASE_					).any(dim=0					)
	if attention_mask is None:
		return input_ids[:, keep_column_mask]
	else:
		return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])



class      SCREAMING_SNAKE_CASE  (				_SCREAMING_SNAKE_CASE						):





	"""simple docstring"""




	def __init__(self     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_="train"     ,     lowerCAmelCase_=None     ,     lowerCAmelCase_=None     ,     lowerCAmelCase_=None     ,     lowerCAmelCase_=""     ,     ):
		super().__init__()
		A_		:						Optional[Any]	    =   Path(lowerCAmelCase_							).joinpath(type_path + """.source"""							)
		A_		:						Union[str, Any]	    =   Path(lowerCAmelCase_							).joinpath(type_path + """.target"""							)
		A_		:						str	    =   self.get_char_lens(self.src_file							)
		A_		:						str	    =   max_source_length
		A_		:						Union[str, Any]	    =   max_target_length
		assert min(self.src_lens							) > 0, f"""found empty line in {self.src_file}"""
		A_		:						Tuple	    =   tokenizer
		A_		:						int	    =   prefix
		if n_obs is not None:
			A_		:						Union[str, Any]	    =   self.src_lens[:n_obs]
		A_		:						List[str]	    =   src_lang
		A_		:						Union[str, Any]	    =   tgt_lang





	def __len__(self							):
		return len(self.src_lens							)





	def __getitem__(self     ,     lowerCAmelCase_							):
		A_		:						Optional[Any]	    =   index + 1  # linecache starts at 1
		A_		:						int	    =   self.prefix + linecache.getline(str(self.src_file							)     ,     lowerCAmelCase_							).rstrip("""\n"""							)
		A_		:						Any	    =   linecache.getline(str(self.tgt_file							)     ,     lowerCAmelCase_							).rstrip("""\n"""							)
		assert source_line, f"""empty source line for index {index}"""
		assert tgt_line, f"""empty tgt line for index {index}"""

		# Need to add eos token manually for T5
		if isinstance(self.tokenizer     ,     lowerCAmelCase_							):
			source_line += self.tokenizer.eos_token
			tgt_line += self.tokenizer.eos_token

		# Pad source and target to the right
		A_		:						Any	    =   (
		    self.tokenizer.question_encoder if isinstance(self.tokenizer     ,     lowerCAmelCase_							) else self.tokenizer
		)
		A_		:						Optional[int]	    =   self.tokenizer.generator if isinstance(self.tokenizer     ,     lowerCAmelCase_							) else self.tokenizer

		A_		:						Tuple	    =   encode_line(lowerCAmelCase_     ,     lowerCAmelCase_     ,     self.max_source_length     ,     """right"""							)
		A_		:						str	    =   encode_line(lowerCAmelCase_     ,     lowerCAmelCase_     ,     self.max_target_length     ,     """right"""							)

		A_		:						int	    =   source_inputs['input_ids'].squeeze()
		A_		:						Tuple	    =   target_inputs['input_ids'].squeeze()
		A_		:						Dict	    =   source_inputs['attention_mask'].squeeze()
		return {
		    "input_ids": source_ids,
		    "attention_mask": src_mask,
		    "decoder_input_ids": target_ids,
		}





	@staticmethod
	def    lowerCamelCase(lowerCAmelCase_							):
		return [len(lowerCAmelCase_							) for x in Path(lowerCAmelCase_							).open().readlines()]





	def    lowerCamelCase(self     ,     lowerCAmelCase_							):
		A_		:						int	    =   torch.stack([x["""input_ids"""] for x in batch]							)
		A_		:						int	    =   torch.stack([x["""attention_mask"""] for x in batch]							)
		A_		:						List[str]	    =   torch.stack([x["""decoder_input_ids"""] for x in batch]							)
		A_		:						Optional[int]	    =   (
		    self.tokenizer.generator.pad_token_id
		    if isinstance(self.tokenizer     ,     lowerCAmelCase_							)
		    else self.tokenizer.pad_token_id
		)
		A_		:						Any	    =   (
		    self.tokenizer.question_encoder.pad_token_id
		    if isinstance(self.tokenizer     ,     lowerCAmelCase_							)
		    else self.tokenizer.pad_token_id
		)
		A_		:						List[str]	    =   trim_batch(lowerCAmelCase_     ,     lowerCAmelCase_							)
		A_		:						List[str]	    =   trim_batch(lowerCAmelCase_     ,     lowerCAmelCase_     ,     attention_mask=lowerCAmelCase_							)
		A_		:						Tuple	    =   {
		    'input_ids': source_ids,
		    'attention_mask': source_mask,
		    'decoder_input_ids': y,
		}
		return batch






_lowerCAmelCase         =							getLogger(__name__)






def 			__UpperCamelCase	(		snake_case__					):
	return list(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE_					)					)






def 			__UpperCamelCase	(		snake_case__					):
	A_		:						Union[str, Any]	    =   get_git_info()
	save_json(SCREAMING_SNAKE_CASE_   ,   os.path.join(SCREAMING_SNAKE_CASE_   ,   """git_log.json"""					)					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__=4   ,   **snake_case__					):
	with open(SCREAMING_SNAKE_CASE_   ,   """w"""					) as f:
		json.dump(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_   ,   indent=SCREAMING_SNAKE_CASE_   ,   **SCREAMING_SNAKE_CASE_					)






def 			__UpperCamelCase	(		snake_case__					):
	with open(SCREAMING_SNAKE_CASE_					) as f:
		return json.load(SCREAMING_SNAKE_CASE_					)






def 			__UpperCamelCase	(		):
	A_		:						int	    =   git.Repo(search_parent_directories=SCREAMING_SNAKE_CASE_					)
	A_		:						Any	    =   {
	    'repo_id': str(SCREAMING_SNAKE_CASE_					),
	    'repo_sha': str(repo.head.object.hexsha					),
	    'repo_branch': str(repo.active_branch					),
	    'hostname': str(socket.gethostname()					),
	}
	return repo_infos






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__					):
	return list(map(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					)					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__					):
	with open(SCREAMING_SNAKE_CASE_   ,   """wb"""					) as f:
		return pickle.dump(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					)






def 			__UpperCamelCase	(		snake_case__					):

	def remove_articles(snake_case__					):
		return re.sub(R"""\b(a|an|the)\b"""   ,   """ """   ,   SCREAMING_SNAKE_CASE_					)

	def white_space_fix(snake_case__					):
		return " ".join(text.split()					)

	def remove_punc(snake_case__					):
		A_		:						Union[str, Any]	    =   set(string.punctuation					)
		return "".join(ch for ch in text if ch not in exclude					)

	def lower(snake_case__					):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE_					)					)					)					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__					):
	A_		:						Any	    =   normalize_answer(SCREAMING_SNAKE_CASE_					).split()
	A_		:						Tuple	    =   normalize_answer(SCREAMING_SNAKE_CASE_					).split()
	A_		:						Optional[Any]	    =   Counter(SCREAMING_SNAKE_CASE_					) & Counter(SCREAMING_SNAKE_CASE_					)
	A_		:						Any	    =   sum(common.values()					)
	if num_same == 0:
		return 0
	A_		:						Optional[Any]	    =   1.0 * num_same / len(SCREAMING_SNAKE_CASE_					)
	A_		:						int	    =   1.0 * num_same / len(SCREAMING_SNAKE_CASE_					)
	A_		:						Union[str, Any]	    =   (2 * precision * recall) / (precision + recall)
	return fa






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__					):
	return normalize_answer(SCREAMING_SNAKE_CASE_					) == normalize_answer(SCREAMING_SNAKE_CASE_					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__					):
	assert len(SCREAMING_SNAKE_CASE_					) == len(SCREAMING_SNAKE_CASE_					)
	A_		:						str	    =   0
	for hypo, pred in zip(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					):
		em += exact_match_score(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					)
	if len(SCREAMING_SNAKE_CASE_					) > 0:
		em /= len(SCREAMING_SNAKE_CASE_					)
	return {"em": em}






def 			__UpperCamelCase	(		snake_case__					):
	return model_prefix.startswith("""rag"""					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__					):
	A_		:						Dict	    =   {p: p for p in extra_params}
	# T5 models don't have `dropout` param, they have `dropout_rate` instead
	A_		:						Optional[int]	    =   'dropout_rate'
	for p in extra_params:
		if getattr(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					):
			if not hasattr(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					) and not hasattr(SCREAMING_SNAKE_CASE_   ,   equivalent_param[p]					):
				logger.info("""config doesn\'t have a `{}` attribute""".format(SCREAMING_SNAKE_CASE_					)					)
				delattr(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					)
				continue
			A_		:						Any	    =   p if hasattr(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					) else equivalent_param[p]
			setattr(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_   ,   getattr(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					)					)
			delattr(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_					)
	return hparams, config









"""simple docstring"""
import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(">=", FSDP_PYTORCH_VERSION):
							import torch.distributed.checkpoint as dist_cp
							from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
							from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
							from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
							from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


_lowerCAmelCase         =							get_logger(__name__)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		A_		:						Optional[int]	    =   model.state_dict()
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			A_		:						int	    =   F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
			A_		:						Union[str, Any]	    =   os.path.join(snake_case__   ,   snake_case__					)
			if accelerator.process_index == 0:
				logger.info(F"""Saving model to {output_model_file}"""					)
				torch.save(snake_case__   ,   snake_case__					)
				logger.info(F"""Model saved to {output_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
			A_		:						List[str]	    =   (
			    F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
			    if model_index == 0
			    else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
			)
			A_		:						List[str]	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Saving model to {output_model_file}"""					)
			torch.save(snake_case__   ,   snake_case__					)
			logger.info(F"""Model saved to {output_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
			A_		:						str	    =   os.path.join(snake_case__   ,   F"""{MODEL_NAME}_{model_index}"""					)
			os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
			logger.info(F"""Saving model to {ckpt_dir}"""					)
			A_		:						Dict	    =   {"""model""": state_dict}

			dist_cp.save_state_dict(
			    state_dict=snake_case__   ,   storage_writer=dist_cp.FileSystemWriter(snake_case__					)   ,   planner=DefaultSavePlanner()   ,   )
			logger.info(F"""Model saved to {ckpt_dir}"""					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	accelerator.wait_for_everyone()
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			if type(snake_case__					) != FSDP and accelerator.process_index != 0:
				if not fsdp_plugin.sync_module_states:
					raise ValueError(
					    """Set the `sync_module_states` flag to `True` so that model states are synced across processes when """
					    """initializing FSDP object"""					)
				return
			A_		:						Tuple	    =   F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
			A_		:						Any	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Loading model from {input_model_file}"""					)
			A_		:						List[Any]	    =   torch.load(snake_case__					)
			logger.info(F"""Model loaded from {input_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
			A_		:						List[str]	    =   (
			    F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
			    if model_index == 0
			    else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
			)
			A_		:						Any	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Loading model from {input_model_file}"""					)
			A_		:						List[str]	    =   torch.load(snake_case__					)
			logger.info(F"""Model loaded from {input_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
			A_		:						Optional[Any]	    =   (
			    os.path.join(snake_case__   ,   F"""{MODEL_NAME}_{model_index}"""					)
			    if F"""{MODEL_NAME}""" not in input_dir
			    else input_dir
			)
			logger.info(F"""Loading model from {ckpt_dir}"""					)
			A_		:						List[str]	    =   {"""model""": model.state_dict()}
			dist_cp.load_state_dict(
			    state_dict=snake_case__   ,   storage_reader=dist_cp.FileSystemReader(snake_case__					)   ,   planner=DefaultLoadPlanner()   ,   )
			A_		:						Tuple	    =   state_dict["""model"""]
			logger.info(F"""Model loaded from {ckpt_dir}"""					)
		model.load_state_dict(snake_case__					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		A_		:						Optional[Any]	    =   FSDP.optim_state_dict(snake_case__   ,   snake_case__					)
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			if accelerator.process_index == 0:
				A_		:						Any	    =   (
				    F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
				)
				A_		:						Optional[int]	    =   os.path.join(snake_case__   ,   snake_case__					)
				logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""					)
				torch.save(snake_case__   ,   snake_case__					)
				logger.info(F"""Optimizer state saved in {output_optimizer_file}"""					)
		else:
			A_		:						Union[str, Any]	    =   os.path.join(snake_case__   ,   F"""{OPTIMIZER_NAME}_{optimizer_index}"""					)
			os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
			logger.info(F"""Saving Optimizer state to {ckpt_dir}"""					)
			dist_cp.save_state_dict(
			    state_dict={"""optimizer""": optim_state}   ,   storage_writer=dist_cp.FileSystemWriter(snake_case__					)   ,   planner=DefaultSavePlanner()   ,   )
			logger.info(F"""Optimizer state saved in {ckpt_dir}"""					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	accelerator.wait_for_everyone()
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			A_		:						List[str]	    =   None
			# below check should work but currently it isn't working (mostly opytorch issue),
			# in the meantime disabling it at the cost of excess memory usage
			# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
			A_		:						Tuple	    =   (
			    F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
			)
			A_		:						int	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""					)
			A_		:						Tuple	    =   torch.load(snake_case__					)
			logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""					)
		else:
			A_		:						str	    =   (
			    os.path.join(snake_case__   ,   F"""{OPTIMIZER_NAME}_{optimizer_index}"""					)
			    if F"""{OPTIMIZER_NAME}""" not in input_dir
			    else input_dir
			)
			logger.info(F"""Loading Optimizer from {ckpt_dir}"""					)
			A_		:						Union[str, Any]	    =   load_sharded_optimizer_state_dict(
			    model_state_dict=model.state_dict()   ,   optimizer_key="""optimizer"""   ,   storage_reader=dist_cp.FileSystemReader(snake_case__					)   ,   )
			A_		:						Tuple	    =   optim_state["""optimizer"""]
			logger.info(F"""Optimizer loaded from {ckpt_dir}"""					)
		A_		:						List[Any]	    =   FSDP.optim_state_dict_to_load(snake_case__   ,   snake_case__   ,   snake_case__					)
		optimizer.load_state_dict(snake_case__					)

