



import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


lowercase_								=     {
    """distilbert""": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    """roberta""": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    """bert""": (BertConfig, BertForMaskedLM, BertTokenizer),
    """gpt2""": (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
			assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
			if args.mlm:
						assert os.path.isfile(args.token_counts  )
						assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
			else:
						assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

			assert args.teacher_type == args.student_type or (
			    args.student_type == "distilbert" and args.teacher_type == "bert"
			)
			assert os.path.isfile(args.student_config  )
			if args.student_pretrained_weights is not None:
						assert os.path.isfile(args.student_pretrained_weights  )

			if args.freeze_token_type_embds:
						assert args.student_type in ["roberta"]

			assert args.alpha_ce >= 0.0
			assert args.alpha_mlm >= 0.0
			assert args.alpha_clm >= 0.0
			assert args.alpha_mse >= 0.0
			assert args.alpha_cos >= 0.0
			assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			if args.student_type == "roberta":
						lowercase__					  =				False
			elif args.student_type == "gpt2":
						lowercase__					  =				False



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			if args.student_type == "roberta":
						lowercase__					  =				False



def __lowerCAmelCase (     ):
			lowercase__					  =				argparse.ArgumentParser(description="Training"  )
			parser.add_argument("--force"						,  action="store_true"						,  help="Overwrite dump_path if it already exists."  )

			parser.add_argument(
			    "--dump_path"						,  type=SCREAMING_SNAKE_CASE_						,  required=SCREAMING_SNAKE_CASE_						,  help="The output directory (log, checkpoints, parameters, etc.)"  )
			parser.add_argument(
			    "--data_file"						,  type=SCREAMING_SNAKE_CASE_						,  required=SCREAMING_SNAKE_CASE_						,  help="The binarized file (tokenized + tokens_to_ids) and grouped by sequence."						,  )

			parser.add_argument(
			    "--student_type"						,  type=SCREAMING_SNAKE_CASE_						,  choices=["distilbert", "roberta", "gpt2"]						,  required=SCREAMING_SNAKE_CASE_						,  help="The student type (DistilBERT, RoBERTa)."						,  )
			parser.add_argument("--student_config"						,  type=SCREAMING_SNAKE_CASE_						,  required=SCREAMING_SNAKE_CASE_						,  help="Path to the student configuration."  )
			parser.add_argument(
			    "--student_pretrained_weights"						,  default=SCREAMING_SNAKE_CASE_						,  type=SCREAMING_SNAKE_CASE_						,  help="Load student initialization checkpoint."  )

			parser.add_argument(
			    "--teacher_type"						,  choices=["bert", "roberta", "gpt2"]						,  required=SCREAMING_SNAKE_CASE_						,  help="Teacher type (BERT, RoBERTa)."  )
			parser.add_argument("--teacher_name"						,  type=SCREAMING_SNAKE_CASE_						,  required=SCREAMING_SNAKE_CASE_						,  help="The teacher model."  )

			parser.add_argument("--temperature"						,  default=2.0						,  type=SCREAMING_SNAKE_CASE_						,  help="Temperature for the softmax temperature."  )
			parser.add_argument(
			    "--alpha_ce"						,  default=0.5						,  type=SCREAMING_SNAKE_CASE_						,  help="Linear weight for the distillation loss. Must be >=0."  )
			parser.add_argument(
			    "--alpha_mlm"						,  default=0.0						,  type=SCREAMING_SNAKE_CASE_						,  help="Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag."						,  )
			parser.add_argument("--alpha_clm"						,  default=0.5						,  type=SCREAMING_SNAKE_CASE_						,  help="Linear weight for the CLM loss. Must be >=0."  )
			parser.add_argument("--alpha_mse"						,  default=0.0						,  type=SCREAMING_SNAKE_CASE_						,  help="Linear weight of the MSE loss. Must be >=0."  )
			parser.add_argument(
			    "--alpha_cos"						,  default=0.0						,  type=SCREAMING_SNAKE_CASE_						,  help="Linear weight of the cosine embedding loss. Must be >=0."  )

			parser.add_argument(
			    "--mlm"						,  action="store_true"						,  help="The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM."  )
			parser.add_argument(
			    "--mlm_mask_prop"						,  default=0.15						,  type=SCREAMING_SNAKE_CASE_						,  help="Proportion of tokens for which we need to make a prediction."						,  )
			parser.add_argument("--word_mask"						,  default=0.8						,  type=SCREAMING_SNAKE_CASE_						,  help="Proportion of tokens to mask out."  )
			parser.add_argument("--word_keep"						,  default=0.1						,  type=SCREAMING_SNAKE_CASE_						,  help="Proportion of tokens to keep."  )
			parser.add_argument("--word_rand"						,  default=0.1						,  type=SCREAMING_SNAKE_CASE_						,  help="Proportion of tokens to randomly replace."  )
			parser.add_argument(
			    "--mlm_smoothing"						,  default=0.7						,  type=SCREAMING_SNAKE_CASE_						,  help="Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec)."						,  )
			parser.add_argument("--token_counts"						,  type=SCREAMING_SNAKE_CASE_						,  help="The token counts in the data_file for MLM."  )

			parser.add_argument(
			    "--restrict_ce_to_mask"						,  action="store_true"						,  help="If true, compute the distillation loss only the [MLM] prediction distribution."						,  )
			parser.add_argument(
			    "--freeze_pos_embs"						,  action="store_true"						,  help="Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only."						,  )
			parser.add_argument(
			    "--freeze_token_type_embds"						,  action="store_true"						,  help="Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only."						,  )

			parser.add_argument("--n_epoch"						,  type=SCREAMING_SNAKE_CASE_						,  default=3						,  help="Number of pass on the whole dataset."  )
			parser.add_argument("--batch_size"						,  type=SCREAMING_SNAKE_CASE_						,  default=5						,  help="Batch size (for each process)."  )
			parser.add_argument(
			    "--group_by_size"						,  action="store_false"						,  help="If true, group sequences that have similar length into the same batch. Default is true."						,  )

			parser.add_argument(
			    "--gradient_accumulation_steps"						,  type=SCREAMING_SNAKE_CASE_						,  default=50						,  help="Gradient accumulation for larger training batches."						,  )
			parser.add_argument("--warmup_prop"						,  default=0.05						,  type=SCREAMING_SNAKE_CASE_						,  help="Linear warmup proportion."  )
			parser.add_argument("--weight_decay"						,  default=0.0						,  type=SCREAMING_SNAKE_CASE_						,  help="Weight decay if we apply some."  )
			parser.add_argument("--learning_rate"						,  default=5e-4						,  type=SCREAMING_SNAKE_CASE_						,  help="The initial learning rate for Adam."  )
			parser.add_argument("--adam_epsilon"						,  default=1e-6						,  type=SCREAMING_SNAKE_CASE_						,  help="Epsilon for Adam optimizer."  )
			parser.add_argument("--max_grad_norm"						,  default=5.0						,  type=SCREAMING_SNAKE_CASE_						,  help="Max gradient norm."  )
			parser.add_argument("--initializer_range"						,  default=0.02						,  type=SCREAMING_SNAKE_CASE_						,  help="Random initialization range."  )

			parser.add_argument(
			    "--fp16"						,  action="store_true"						,  help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"						,  )
			parser.add_argument(
			    "--fp16_opt_level"						,  type=SCREAMING_SNAKE_CASE_						,  default="O1"						,  help=(
			        "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
			        "See details at https://nvidia.github.io/apex/amp.html"
			    )						,  )
			parser.add_argument("--n_gpu"						,  type=SCREAMING_SNAKE_CASE_						,  default=1						,  help="Number of GPUs in the node."  )
			parser.add_argument("--local_rank"						,  type=SCREAMING_SNAKE_CASE_						,  default=-1						,  help="Distributed training - Local rank"  )
			parser.add_argument("--seed"						,  type=SCREAMING_SNAKE_CASE_						,  default=56						,  help="Random seed"  )

			parser.add_argument("--log_interval"						,  type=SCREAMING_SNAKE_CASE_						,  default=500						,  help="Tensorboard logging interval."  )
			parser.add_argument("--checkpoint_interval"						,  type=SCREAMING_SNAKE_CASE_						,  default=4000						,  help="Checkpoint interval."  )
			lowercase__					  =				parser.parse_args()
			sanity_checks(SCREAMING_SNAKE_CASE_  )

			# ARGS #
			init_gpu_params(SCREAMING_SNAKE_CASE_  )
			set_seed(SCREAMING_SNAKE_CASE_  )
			if args.is_master:
						if os.path.exists(args.dump_path  ):
									if not args.force:
												raise ValueError(
												    f'''Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'''
												    " itUse `--force` if you want to overwrite it"  )
									else:
												shutil.rmtree(args.dump_path  )

						if not os.path.exists(args.dump_path  ):
									os.makedirs(args.dump_path  )
						logger.info(f'''Experiment will be dumped and logged in {args.dump_path}'''  )

						# SAVE PARAMS #
						logger.info(f'''Param: {args}'''  )
						with open(os.path.join(args.dump_path						,  "parameters.json"  )						,  "w"  ) as f:
									json.dump(vars(SCREAMING_SNAKE_CASE_  )						,  SCREAMING_SNAKE_CASE_						,  indent=4  )
						git_log(args.dump_path  )

			lowercase__  ,       lowercase__  ,       lowercase__					  =				MODEL_CLASSES[args.student_type]
			lowercase__  ,       lowercase__  ,       lowercase__					  =				MODEL_CLASSES[args.teacher_type]

			# TOKENIZER #
			lowercase__					  =				teacher_tokenizer_class.from_pretrained(args.teacher_name  )
			lowercase__					  =				{}
			for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
						lowercase__					  =				tokenizer.all_special_tokens.index(SCREAMING_SNAKE_CASE_  )
						lowercase__					  =				tokenizer.all_special_ids[idx]
			logger.info(f'''Special tokens {special_tok_ids}'''  )
			lowercase__					  =				special_tok_ids
			lowercase__					  =				tokenizer.max_model_input_sizes[args.teacher_name]

			# DATA LOADER #
			logger.info(f'''Loading data from {args.data_file}'''  )
			with open(args.data_file						,  "rb"  ) as fp:
						lowercase__					  =				pickle.load(SCREAMING_SNAKE_CASE_  )

			if args.mlm:
						logger.info(f'''Loading token counts from {args.token_counts} (already pre-computed)'''  )
						with open(args.token_counts						,  "rb"  ) as fp:
									lowercase__					  =				pickle.load(SCREAMING_SNAKE_CASE_  )

						lowercase__					  =				np.maximum(SCREAMING_SNAKE_CASE_						,  1  ) ** -args.mlm_smoothing
						for idx in special_tok_ids.values():
									lowercase__					  =				0.0  # do not predict special tokens
						lowercase__					  =				torch.from_numpy(SCREAMING_SNAKE_CASE_  )
			else:
						lowercase__					  =				None

			lowercase__					  =				LmSeqsDataset(params=SCREAMING_SNAKE_CASE_						,  data=SCREAMING_SNAKE_CASE_  )
			logger.info("Data loader created."  )

			# STUDENT #
			logger.info(f'''Loading student config from {args.student_config}'''  )
			lowercase__					  =				student_config_class.from_pretrained(args.student_config  )
			lowercase__					  =				True

			if args.student_pretrained_weights is not None:
						logger.info(f'''Loading pretrained weights from {args.student_pretrained_weights}'''  )
						lowercase__					  =				student_model_class.from_pretrained(args.student_pretrained_weights						,  config=SCREAMING_SNAKE_CASE_  )
			else:
						lowercase__					  =				student_model_class(SCREAMING_SNAKE_CASE_  )

			if args.n_gpu > 0:
						student.to(f'''cuda:{args.local_rank}'''  )
			logger.info("Student loaded."  )

			# TEACHER #
			lowercase__					  =				teacher_model_class.from_pretrained(args.teacher_name						,  output_hidden_states=SCREAMING_SNAKE_CASE_  )
			if args.n_gpu > 0:
						teacher.to(f'''cuda:{args.local_rank}'''  )
			logger.info(f'''Teacher loaded from {args.teacher_name}.'''  )

			# FREEZING #
			if args.freeze_pos_embs:
						freeze_pos_embeddings(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			if args.freeze_token_type_embds:
						freeze_token_type_embeddings(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )

			# SANITY CHECKS #
			assert student.config.vocab_size == teacher.config.vocab_size
			assert student.config.hidden_size == teacher.config.hidden_size
			assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
			if args.mlm:
						assert token_probs.size(0  ) == stu_architecture_config.vocab_size

			# DISTILLER #
			torch.cuda.empty_cache()
			lowercase__					  =				Distiller(
			    params=SCREAMING_SNAKE_CASE_						,  dataset=SCREAMING_SNAKE_CASE_						,  token_probs=SCREAMING_SNAKE_CASE_						,  student=SCREAMING_SNAKE_CASE_						,  teacher=SCREAMING_SNAKE_CASE_  )
			distiller.train()
			logger.info("Let's go get some drinks."  )


if __name__ == "__main__":
							main()









import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


lowercase_								=     logging.get_logger(__name__)


lowercase_								=     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase_								=     {
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

lowercase_								=     {
    """allenai/led-base-16384""": 1_6384,
}
class 				_snake_case			(					lowercase__):
				UpperCamelCase__				:					int     		=VOCAB_FILES_NAMES
				UpperCamelCase__				:					Any     		=PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase__				:					Dict     		=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase__				:					List[Any]     		=LEDTokenizer
				UpperCamelCase__				:					Tuple     		=["""input_ids""", """attention_mask"""]




				def __init__(						self					:						Optional[Any],					__lowercase					:						Optional[Any]=None,					__lowercase					:						Dict=None,					__lowercase					:						Tuple=None,					__lowercase					:						Union[str, Any]="replace",					__lowercase					:						Tuple="<s>",					__lowercase					:						Optional[Any]="</s>",					__lowercase					:						Tuple="</s>",					__lowercase					:						List[str]="<s>",					__lowercase					:						Tuple="<unk>",					__lowercase					:						Dict="<pad>",					__lowercase					:						Dict="<mask>",					__lowercase					:						Any=False,					__lowercase					:						Any=True,					**__lowercase					:						List[Any],					):
							super().__init__(
							    __lowercase,					__lowercase,					tokenizer_file=__lowercase,					errors=__lowercase,					bos_token=__lowercase,					eos_token=__lowercase,					sep_token=__lowercase,					cls_token=__lowercase,					unk_token=__lowercase,					pad_token=__lowercase,					mask_token=__lowercase,					add_prefix_space=__lowercase,					trim_offsets=__lowercase,					**__lowercase,					)

							lowercase__					  =				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
							if pre_tok_state.get("add_prefix_space",					__lowercase						) != add_prefix_space:
										lowercase__					  =				getattr(__lowercase,					pre_tok_state.pop("type"						)						)
										lowercase__					  =				add_prefix_space
										lowercase__					  =				pre_tok_class(**__lowercase						)

							lowercase__					  =				add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							lowercase__					  =				"post_processor"
							lowercase__					  =				getattr(self.backend_tokenizer,					__lowercase,					__lowercase						)
							if tokenizer_component_instance:
										lowercase__					  =				json.loads(tokenizer_component_instance.__getstate__()						)

										# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
										if "sep" in state:
													lowercase__					  =				tuple(state["sep"]						)
										if "cls" in state:
													lowercase__					  =				tuple(state["cls"]						)

										lowercase__					  =				False

										if state.get("add_prefix_space",					__lowercase						) != add_prefix_space:
													lowercase__					  =				add_prefix_space
													lowercase__					  =				True

										if state.get("trim_offsets",					__lowercase						) != trim_offsets:
													lowercase__					  =				trim_offsets
													lowercase__					  =				True

										if changes_to_apply:
													lowercase__					  =				getattr(__lowercase,					state.pop("type"						)						)
													lowercase__					  =				component_class(**__lowercase						)
													setattr(self.backend_tokenizer,					__lowercase,					__lowercase						)




				@property
				# Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
				def 							A__      (						self					:						str						):
							if self._mask_token is None:
										if self.verbose:
													logger.error("Using mask_token, but it is not set yet."						)
										return None
							return str(self._mask_token						)




				@mask_token.setter
				def 							A__      (						self					:						Optional[int],					__lowercase					:						Dict						):
							lowercase__					  =				AddedToken(__lowercase,					lstrip=__lowercase,					rstrip=__lowercase						) if isinstance(__lowercase,					__lowercase						) else value
							lowercase__					  =				value




				def 							A__      (						self					:						Any,					*__lowercase					:						List[Any],					**__lowercase					:						Optional[Any]						):
							lowercase__					  =				kwargs.get("is_split_into_words",					__lowercase						)

							if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."						)

							return super()._batch_encode_plus(*__lowercase,					**__lowercase						)




				def 							A__      (						self					:						int,					*__lowercase					:						Union[str, Any],					**__lowercase					:						List[str]						):
							lowercase__					  =				kwargs.get("is_split_into_words",					__lowercase						)

							if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."						)

							return super()._encode_plus(*__lowercase,					**__lowercase						)




				def 							A__      (						self					:						Optional[Any],					__lowercase					:						str,					__lowercase					:						Optional[str] = None						):
							lowercase__					  =				self._tokenizer.model.save(__lowercase,					name=__lowercase						)
							return tuple(__lowercase						)




				def 							A__      (						self					:						List[str],					__lowercase					:						int,					__lowercase					:						Optional[int]=None						):
							lowercase__					  =				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
										return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




				def 							A__      (						self					:						int,					__lowercase					:						List[int],					__lowercase					:						Optional[List[int]] = None						):
							lowercase__					  =				[self.sep_token_id]
							lowercase__					  =				[self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep						) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]




				def 							A__      (						self					:						Union[str, Any],					__lowercase					:						Union[Dict[str, EncodedInput], BatchEncoding],					__lowercase					:						Optional[int] = None,					__lowercase					:						PaddingStrategy = PaddingStrategy.DO_NOT_PAD,					__lowercase					:						Optional[int] = None,					__lowercase					:						Optional[bool] = None,					):
							lowercase__					  =				super()._pad(
							    encoded_inputs=__lowercase,					max_length=__lowercase,					padding_strategy=__lowercase,					pad_to_multiple_of=__lowercase,					return_attention_mask=__lowercase,					)

							# Load from model defaults
							if return_attention_mask is None:
										lowercase__					  =				"attention_mask" in self.model_input_names

							if return_attention_mask and "global_attention_mask" in encoded_inputs:
										lowercase__					  =				encoded_inputs[self.model_input_names[0]]
										# `global_attention_mask` need to have the same length as other (sequential) inputs.
										lowercase__					  =				len(encoded_inputs["global_attention_mask"]						) != len(__lowercase						)

										if needs_to_be_padded:
													lowercase__					  =				len(__lowercase						) - len(encoded_inputs["global_attention_mask"]						)

													if self.padding_side == "right":
																# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																lowercase__					  =				(
																    encoded_inputs["global_attention_mask"] + [-1] * difference
																)
													elif self.padding_side == "left":
																lowercase__					  =				[-1] * difference + encoded_inputs[
																    "global_attention_mask"
																]
													else:
																raise ValueError("Invalid padding strategy:" + str(self.padding_side						)						)

							return encoded_inputs




