






import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


__A      =				logging.getLogger(__name__)





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							A_						  =					"sequence-classification"

							def __init__(  self:   str		,     __A:   Union[str, Any]   )    ->				List[str]:
												if type(__A   ) == dict:
																	_A								= Namespace(**__A   )
												_A								= glue_output_modes[hparams.task]
												_A								= glue_tasks_num_labels[hparams.task]

												super().__init__(__A		,     __A		,     self.mode   )

							def       __A (  self:   Optional[Any]		,     **__A:   Union[str, Any]   )    ->				Optional[int]:
												return self.model(**__A   )

							def       __A (  self:   Any		,     __A:   Union[str, Any]		,     __A:   int   )    ->				Optional[Any]:
												_A								= {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}

												if self.config.model_type not in ["distilbert", "bart"]:
																	_A								= batch[2] if self.config.model_type in ['''bert''', '''xlnet''', '''albert'''] else None

												_A								= self(**__A   )
												_A								= outputs[0]

												_A								= self.trainer.lr_schedulers[0]['''scheduler''']
												_A								= {'''loss''': loss, '''rate''': lr_scheduler.get_last_lr()[-1]}
												return {"loss": loss, "log": tensorboard_logs}

							def       __A (  self:   List[str]   )    ->				Dict:
												_A								= self.hparams
												_A								= processors[args.task]()
												_A								= processor.get_labels()

												for mode in ["train", "dev"]:
																	_A								= self._feature_file(__A   )
																	if os.path.exists(__A   ) and not args.overwrite_cache:
																						logger.info('''Loading features from cached file %s'''		,     __A   )
																	else:
																						logger.info('''Creating features from dataset file at %s'''		,     args.data_dir   )
																						_A								= (
																						    processor.get_dev_examples(args.data_dir   )
																						    if mode == '''dev'''
																						    else processor.get_train_examples(args.data_dir   )
																						)
																						_A								= convert_examples_to_features(
																						    __A		,     self.tokenizer		,     max_length=args.max_seq_length		,     label_list=self.labels		,     output_mode=args.glue_output_mode		,     )
																						logger.info('''Saving features into cached file %s'''		,     __A   )
																						torch.save(__A		,     __A   )

							def       __A (  self:   List[str]		,     __A:   str		,     __A:   int		,     __A:   bool = False   )    ->				DataLoader:
												_A								= '''dev''' if mode == '''test''' else mode

												_A								= self._feature_file(__A   )
												logger.info('''Loading features from cached file %s'''		,     __A   )
												_A								= torch.load(__A   )
												_A								= torch.tensor([f.input_ids for f in features]		,     dtype=torch.long   )
												_A								= torch.tensor([f.attention_mask for f in features]		,     dtype=torch.long   )
												_A								= torch.tensor([f.token_type_ids for f in features]		,     dtype=torch.long   )
												if self.hparams.glue_output_mode == "classification":
																	_A								= torch.tensor([f.label for f in features]		,     dtype=torch.long   )
												elif self.hparams.glue_output_mode == "regression":
																	_A								= torch.tensor([f.label for f in features]		,     dtype=torch.float   )

												return DataLoader(
												    TensorDataset(__A		,     __A		,     __A		,     __A   )		,     batch_size=__A		,     shuffle=__A		,     )

							def       __A (  self:   List[str]		,     __A:   str		,     __A:   Tuple   )    ->				str:
												_A								= {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}

												if self.config.model_type not in ["distilbert", "bart"]:
																	_A								= batch[2] if self.config.model_type in ['''bert''', '''xlnet''', '''albert'''] else None

												_A								= self(**__A   )
												_A							,_A								= outputs[:2]
												_A								= logits.detach().cpu().numpy()
												_A								= inputs['''labels'''].detach().cpu().numpy()

												return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}

							def       __A (  self:   str		,     __A:   Dict   )    ->				tuple:
												_A								= torch.stack([x['''val_loss'''] for x in outputs]   ).mean().detach().cpu().item()
												_A								= np.concatenate([x['''pred'''] for x in outputs]		,     axis=0   )

												if self.hparams.glue_output_mode == "classification":
																	_A								= np.argmax(__A		,     axis=1   )
												elif self.hparams.glue_output_mode == "regression":
																	_A								= np.squeeze(__A   )

												_A								= np.concatenate([x['''target'''] for x in outputs]		,     axis=0   )
												_A								= [[] for _ in range(out_label_ids.shape[0]   )]
												_A								= [[] for _ in range(out_label_ids.shape[0]   )]

												_A								= {**{'''val_loss''': val_loss_mean}, **compute_metrics(self.hparams.task		,     __A		,     __A   )}

												_A								= dict(results.items()   )
												_A								= results
												return ret, preds_list, out_label_list

							def       __A (  self:   Any		,     __A:   list   )    ->				dict:
												_A							,_A							,_A								= self._eval_end(__A   )
												_A								= ret['''log''']
												return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}

							def       __A (  self:   int		,     __A:   Union[str, Any]   )    ->				dict:
												_A							,_A							,_A								= self._eval_end(__A   )
												_A								= ret['''log''']
												# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
												return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}



							@staticmethod
							def       __A (  __A:   Optional[Any]		,     __A:   Optional[Any]   )    ->				Optional[Any]:
												BaseTransformer.add_model_specific_args(__A		,     __A   )
												parser.add_argument(
												    '''--max_seq_length'''		,     default=1_28		,     type=__A		,     help=(
												        '''The maximum total input sequence length after tokenization. Sequences longer '''
												        '''than this will be truncated, sequences shorter will be padded.'''
												    )		,     )

												parser.add_argument(
												    '''--task'''		,     default=''''''		,     type=__A		,     required=__A		,     help='''The GLUE task to run'''		,     )
												parser.add_argument(
												    '''--gpus'''		,     default=0		,     type=__A		,     help='''The number of GPUs allocated for this, it is by default 0 meaning none'''		,     )

												parser.add_argument(
												    '''--overwrite_cache'''		,     action='''store_true'''		,     help='''Overwrite the cached training and evaluation sets'''   )

												return parser




def      __A					(     ):

					'''simple docstring'''







					_A								= argparse.ArgumentParser()
					add_generic_args(_lowercase					,			os.getcwd() )
					_A								= GLUETransformer.add_model_specific_args(_lowercase					,			os.getcwd() )
					_A								= parser.parse_args()

					# If output_dir not provided, a folder will be generated in pwd
					if args.output_dir is None:
										_A								= os.path.join(
										    '''./results'''					,			f"""{args.task}_{time.strftime("%Y%m%d_%H%M%S" )}"""					,			)
										os.makedirs(args.output_dir )

					_A								= GLUETransformer(_lowercase )
					_A								= generic_train(_lowercase					,			_lowercase )

					# Optionally, predict on dev set and write to output_dir
					if args.do_predict:
										_A								= sorted(glob.glob(os.path.join(args.output_dir					,			'''checkpoint-epoch=*.ckpt''' )					,			recursive=_lowercase ) )
										_A								= model.load_from_checkpoint(checkpoints[-1] )
										return trainer.test(_lowercase )


if __name__ == "__main__":
				main()











import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__A      =				logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm
				import torch_xla.debug.metrics as met





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							def __init__(  self:   int		,     *__A:   str		,     __A:   List[Any]=None		,     __A:   Union[str, Any]=None		,     __A:   List[Any]=None		,     **__A:   int   )    ->				List[Any]:
												super().__init__(*__A		,     **__A   )
												_A								= eval_examples
												_A								= post_process_function
												_A								= quant_trainer_args
												_A								= 1_28  # default number of calibration samples

							def       __A (  self:   Union[str, Any]		,     __A:   List[Any]=None   )    ->				Optional[Any]:
												if calib_dataset is None and self.calib_dataset is None:
																	raise ValueError('''Trainer: calibration requires an calib_dataset.'''   )
												_A								= calib_dataset if calib_dataset is not None else self.calib_dataset

												_A								= self._remove_unused_columns(__A		,     description='''Calibration'''   )

												return DataLoader(
												    __A		,     batch_size=self.args.eval_batch_size		,     collate_fn=self.data_collator		,     drop_last=self.args.dataloader_drop_last		,     num_workers=self.args.dataloader_num_workers		,     pin_memory=self.args.dataloader_pin_memory		,     shuffle=__A		,     )

							def       __A (  self:   List[Any]		,     __A:   Any=None   )    ->				Optional[int]:
												_A								= self.train_dataset if calib_dataset is None else calib_dataset
												_A								= self.get_calib_dataloader(__A   )

												_A								= self.model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args		,     calib=__A   )
												model.eval()
												quant_trainer.enable_calibration(__A   )

												logger.info('''***** Running calibration *****'''   )
												logger.info(f"""  Num examples = {self.calib_num}"""   )
												logger.info(f"""  Batch size = {calib_dataloader.batch_size}"""   )

												for step, inputs in enumerate(__A   ):
																	# Prediction step
																	_A							,_A							,_A								= self.prediction_step(__A		,     __A		,     prediction_loss_only=__A   )
																	if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
																						break

												quant_trainer.finish_calibration(__A		,     self.quant_trainer_args   )
												_A								= model

							def       __A (  self:   Any		,     __A:   Dict=None		,     __A:   Tuple=None		,     __A:   List[Any]=None		,     __A:   str = "eval"   )    ->				int:
												_A								= self.eval_dataset if eval_dataset is None else eval_dataset
												_A								= self.get_eval_dataloader(__A   )
												_A								= self.eval_examples if eval_examples is None else eval_examples

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Evaluation'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is not None and self.compute_metrics is not None:
																	_A								= self.post_process_function(__A		,     __A		,     output.predictions   )
																	_A								= self.compute_metrics(__A   )

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()   ):
																						if not key.startswith(f"""{metric_key_prefix}_"""   ):
																											_A								= metrics.pop(__A   )

																	self.log(__A   )
												else:
																	_A								= {}

												if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report()   )

												_A								= self.callback_handler.on_evaluate(self.args		,     self.state		,     self.control		,     __A   )
												return metrics

							def       __A (  self:   Union[str, Any]		,     __A:   Optional[int]		,     __A:   int		,     __A:   List[Any]=None		,     __A:   str = "test"   )    ->				Union[str, Any]:
												_A								= self.get_test_dataloader(__A   )

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Prediction'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is None or self.compute_metrics is None:
																	return output

												_A								= self.post_process_function(__A		,     __A		,     output.predictions		,     '''predict'''   )
												_A								= self.compute_metrics(__A   )

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()   ):
																	if not key.startswith(f"""{metric_key_prefix}_"""   ):
																						_A								= metrics.pop(__A   )

												return PredictionOutput(predictions=predictions.predictions		,     label_ids=predictions.label_ids		,     metrics=__A   )



							def       __A (  self:   Tuple		,     __A:   Optional[Any]="./"   )    ->				List[str]:
												_A								= self.eval_dataset
												_A								= self.get_eval_dataloader(__A   )

												_A								= next(iter(__A   )   )

												# saving device - to make it consistent
												_A								= torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''   )

												# convert to tuple
												_A								= tuple(v.to(__A   ) for k, v in batch.items()   )

												logger.info('''Converting model to be onnx compatible'''   )
												from pytorch_quantization.nn import TensorQuantizer

												_A								= True

												_A								= self.model.to(__A   )

												model.eval()
												model.float()

												_A								= model.module if hasattr(__A		,     '''module'''   ) else model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args   )

												_A								= os.path.join(__A		,     '''model.onnx'''   )
												logger.info(f"""exporting model to {output_model_file}"""   )

												_A								= {0: '''batch_size''', 1: '''seq_len'''}

												torch.onnx.export(
												    __A		,     __A		,     __A		,     export_params=__A		,     opset_version=13		,     do_constant_folding=__A		,     input_names=['''input_ids''', '''attention_mask''', '''token_type_ids''']		,     output_names=['''output_start_logits''', '''output_end_logits''']		,     dynamic_axes={
												        '''input_ids''': axes,
												        '''attention_mask''': axes,
												        '''token_type_ids''': axes,
												        '''output_start_logits''': axes,
												        '''output_end_logits''': axes,
												    }		,     verbose=__A		,     )
												logger.info('''onnx export finished'''   )



