






import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text




def      __A					(     _lowercase ):

					'''simple docstring'''







					_A								= []
					for line in lines:
										_A								= re.sub(R'''#.*'''					,			''''''					,			_lowercase )  # remove comments
										if line:
															filtered_lines.append(_lowercase )
					_A								= '''\n'''.join(_lowercase )

					# Make a hash from all this code
					_A								= full_str.encode('''utf-8''' )
					return shaaaa(_lowercase ).hexdigest()


# get importable module names and hash for caching
__A      =   {
    """csv""": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    """json""": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    """pandas""": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    """parquet""": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    """arrow""": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    """text""": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    """imagefolder""": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    """audiofolder""": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
__A      =   {
    """.csv""": ("""csv""", {}),
    """.tsv""": ("""csv""", {"""sep""": """\t"""}),
    """.json""": ("""json""", {}),
    """.jsonl""": ("""json""", {}),
    """.parquet""": ("""parquet""", {}),
    """.arrow""": ("""arrow""", {}),
    """.txt""": ("""text""", {}),
}
_EXTENSION_TO_MODULE.update({ext: ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
__A      =   {"""imagefolder""", """audiofolder"""}

# Used to filter data files based on extensions given a module name
__A      =   {}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
				_MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append('.zip')


_MODULE_TO_EXTENSIONS["audiofolder"].append('.zip')











import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__A      =				logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm
				import torch_xla.debug.metrics as met





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							def __init__(  self:   int		,     *__A:   str		,     __A:   List[Any]=None		,     __A:   Union[str, Any]=None		,     __A:   List[Any]=None		,     **__A:   int   )    ->				List[Any]:
												super().__init__(*__A		,     **__A   )
												_A								= eval_examples
												_A								= post_process_function
												_A								= quant_trainer_args
												_A								= 1_28  # default number of calibration samples

							def       __A (  self:   Union[str, Any]		,     __A:   List[Any]=None   )    ->				Optional[Any]:
												if calib_dataset is None and self.calib_dataset is None:
																	raise ValueError('''Trainer: calibration requires an calib_dataset.'''   )
												_A								= calib_dataset if calib_dataset is not None else self.calib_dataset

												_A								= self._remove_unused_columns(__A		,     description='''Calibration'''   )

												return DataLoader(
												    __A		,     batch_size=self.args.eval_batch_size		,     collate_fn=self.data_collator		,     drop_last=self.args.dataloader_drop_last		,     num_workers=self.args.dataloader_num_workers		,     pin_memory=self.args.dataloader_pin_memory		,     shuffle=__A		,     )

							def       __A (  self:   List[Any]		,     __A:   Any=None   )    ->				Optional[int]:
												_A								= self.train_dataset if calib_dataset is None else calib_dataset
												_A								= self.get_calib_dataloader(__A   )

												_A								= self.model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args		,     calib=__A   )
												model.eval()
												quant_trainer.enable_calibration(__A   )

												logger.info('''***** Running calibration *****'''   )
												logger.info(f"""  Num examples = {self.calib_num}"""   )
												logger.info(f"""  Batch size = {calib_dataloader.batch_size}"""   )

												for step, inputs in enumerate(__A   ):
																	# Prediction step
																	_A							,_A							,_A								= self.prediction_step(__A		,     __A		,     prediction_loss_only=__A   )
																	if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
																						break

												quant_trainer.finish_calibration(__A		,     self.quant_trainer_args   )
												_A								= model

							def       __A (  self:   Any		,     __A:   Dict=None		,     __A:   Tuple=None		,     __A:   List[Any]=None		,     __A:   str = "eval"   )    ->				int:
												_A								= self.eval_dataset if eval_dataset is None else eval_dataset
												_A								= self.get_eval_dataloader(__A   )
												_A								= self.eval_examples if eval_examples is None else eval_examples

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Evaluation'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is not None and self.compute_metrics is not None:
																	_A								= self.post_process_function(__A		,     __A		,     output.predictions   )
																	_A								= self.compute_metrics(__A   )

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()   ):
																						if not key.startswith(f"""{metric_key_prefix}_"""   ):
																											_A								= metrics.pop(__A   )

																	self.log(__A   )
												else:
																	_A								= {}

												if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report()   )

												_A								= self.callback_handler.on_evaluate(self.args		,     self.state		,     self.control		,     __A   )
												return metrics

							def       __A (  self:   Union[str, Any]		,     __A:   Optional[int]		,     __A:   int		,     __A:   List[Any]=None		,     __A:   str = "test"   )    ->				Union[str, Any]:
												_A								= self.get_test_dataloader(__A   )

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Prediction'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is None or self.compute_metrics is None:
																	return output

												_A								= self.post_process_function(__A		,     __A		,     output.predictions		,     '''predict'''   )
												_A								= self.compute_metrics(__A   )

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()   ):
																	if not key.startswith(f"""{metric_key_prefix}_"""   ):
																						_A								= metrics.pop(__A   )

												return PredictionOutput(predictions=predictions.predictions		,     label_ids=predictions.label_ids		,     metrics=__A   )



							def       __A (  self:   Tuple		,     __A:   Optional[Any]="./"   )    ->				List[str]:
												_A								= self.eval_dataset
												_A								= self.get_eval_dataloader(__A   )

												_A								= next(iter(__A   )   )

												# saving device - to make it consistent
												_A								= torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''   )

												# convert to tuple
												_A								= tuple(v.to(__A   ) for k, v in batch.items()   )

												logger.info('''Converting model to be onnx compatible'''   )
												from pytorch_quantization.nn import TensorQuantizer

												_A								= True

												_A								= self.model.to(__A   )

												model.eval()
												model.float()

												_A								= model.module if hasattr(__A		,     '''module'''   ) else model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args   )

												_A								= os.path.join(__A		,     '''model.onnx'''   )
												logger.info(f"""exporting model to {output_model_file}"""   )

												_A								= {0: '''batch_size''', 1: '''seq_len'''}

												torch.onnx.export(
												    __A		,     __A		,     __A		,     export_params=__A		,     opset_version=13		,     do_constant_folding=__A		,     input_names=['''input_ids''', '''attention_mask''', '''token_type_ids''']		,     output_names=['''output_start_logits''', '''output_end_logits''']		,     dynamic_axes={
												        '''input_ids''': axes,
												        '''attention_mask''': axes,
												        '''token_type_ids''': axes,
												        '''output_start_logits''': axes,
												        '''output_end_logits''': axes,
												    }		,     verbose=__A		,     )
												logger.info('''onnx export finished'''   )



