






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__A      =   logging.get_logger(__name__)

__A      =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__A      =   {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__A      =   {
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


__A      =   {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							A_						  =					VOCAB_FILES_NAMES
							A_						  =					PRETRAINED_VOCAB_FILES_MAP
							A_						  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_						  =					PRETRAINED_INIT_CONFIGURATION
							A_						  =					RoFormerTokenizer

							def __init__(  self:   Any		,     __A:   Union[str, Any]=None		,     __A:   Dict=None		,     __A:   Dict=True		,     __A:   Union[str, Any]="[UNK]"		,     __A:   List[Any]="[SEP]"		,     __A:   List[str]="[PAD]"		,     __A:   List[str]="[CLS]"		,     __A:   List[str]="[MASK]"		,     __A:   int=True		,     __A:   List[Any]=None		,     **__A:   int		,     )    ->				int:
												super().__init__(
												    __A		,     tokenizer_file=__A		,     do_lower_case=__A		,     unk_token=__A		,     sep_token=__A		,     pad_token=__A		,     cls_token=__A		,     mask_token=__A		,     tokenize_chinese_chars=__A		,     strip_accents=__A		,     **__A		,     )

												_A								= json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
												if (
												    pre_tok_state.get('''lowercase'''		,     __A   ) != do_lower_case
												    or pre_tok_state.get('''strip_accents'''		,     __A   ) != strip_accents
												):
																	_A								= getattr(__A		,     pre_tok_state.pop('''type'''   )   )
																	_A								= do_lower_case
																	_A								= strip_accents
																	_A								= pre_tok_class(**__A   )

												_A								= do_lower_case

							def __getstate__(  self:   Union[str, Any]   )    ->				str:
												_A								= self.__dict__.copy()
												_A								= BertPreTokenizer()
												return state

							def __setstate__(  self:   Union[str, Any]		,     __A:   Union[str, Any]   )    ->				Optional[int]:
												_A								= d
												_A								= self.__dict__['''_tokenizer'''].get_vocab()
												_A								= PreTokenizer.custom(JiebaPreTokenizer(__A   )   )

							def       __A (  self:   str		,     __A:   str		,     __A:   str=None   )    ->				int:
												_A								= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

												return output

							def       __A (  self:   Tuple		,     __A:   List[int]		,     __A:   Optional[List[int]] = None   )    ->				List[int]:
												_A								= [self.sep_token_id]
												_A								= [self.cls_token_id]
												if token_ids_a is None:
																	return len(cls + token_ids_a + sep   ) * [0]
												return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]

							def       __A (  self:   List[Any]		,     __A:   str		,     __A:   Optional[str] = None   )    ->				Tuple[str]:
												_A								= self._tokenizer.model.save(__A		,     name=__A   )
												return tuple(__A   )



							def       __A (  self:   List[str]		,     __A:   int		,     __A:   List[Any]=None		,     __A:   Union[str, Any]=None		,     __A:   List[str]=False		,     **__A:   Optional[Any]		,     )    ->				Dict:
												_A								= BertPreTokenizer()
												return super().save_pretrained(__A		,     __A		,     __A		,     __A		,     **__A   )











import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__A      =				logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm
				import torch_xla.debug.metrics as met





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							def __init__(  self:   int		,     *__A:   str		,     __A:   List[Any]=None		,     __A:   Union[str, Any]=None		,     __A:   List[Any]=None		,     **__A:   int   )    ->				List[Any]:
												super().__init__(*__A		,     **__A   )
												_A								= eval_examples
												_A								= post_process_function
												_A								= quant_trainer_args
												_A								= 1_28  # default number of calibration samples

							def       __A (  self:   Union[str, Any]		,     __A:   List[Any]=None   )    ->				Optional[Any]:
												if calib_dataset is None and self.calib_dataset is None:
																	raise ValueError('''Trainer: calibration requires an calib_dataset.'''   )
												_A								= calib_dataset if calib_dataset is not None else self.calib_dataset

												_A								= self._remove_unused_columns(__A		,     description='''Calibration'''   )

												return DataLoader(
												    __A		,     batch_size=self.args.eval_batch_size		,     collate_fn=self.data_collator		,     drop_last=self.args.dataloader_drop_last		,     num_workers=self.args.dataloader_num_workers		,     pin_memory=self.args.dataloader_pin_memory		,     shuffle=__A		,     )

							def       __A (  self:   List[Any]		,     __A:   Any=None   )    ->				Optional[int]:
												_A								= self.train_dataset if calib_dataset is None else calib_dataset
												_A								= self.get_calib_dataloader(__A   )

												_A								= self.model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args		,     calib=__A   )
												model.eval()
												quant_trainer.enable_calibration(__A   )

												logger.info('''***** Running calibration *****'''   )
												logger.info(f"""  Num examples = {self.calib_num}"""   )
												logger.info(f"""  Batch size = {calib_dataloader.batch_size}"""   )

												for step, inputs in enumerate(__A   ):
																	# Prediction step
																	_A							,_A							,_A								= self.prediction_step(__A		,     __A		,     prediction_loss_only=__A   )
																	if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
																						break

												quant_trainer.finish_calibration(__A		,     self.quant_trainer_args   )
												_A								= model

							def       __A (  self:   Any		,     __A:   Dict=None		,     __A:   Tuple=None		,     __A:   List[Any]=None		,     __A:   str = "eval"   )    ->				int:
												_A								= self.eval_dataset if eval_dataset is None else eval_dataset
												_A								= self.get_eval_dataloader(__A   )
												_A								= self.eval_examples if eval_examples is None else eval_examples

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Evaluation'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is not None and self.compute_metrics is not None:
																	_A								= self.post_process_function(__A		,     __A		,     output.predictions   )
																	_A								= self.compute_metrics(__A   )

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()   ):
																						if not key.startswith(f"""{metric_key_prefix}_"""   ):
																											_A								= metrics.pop(__A   )

																	self.log(__A   )
												else:
																	_A								= {}

												if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report()   )

												_A								= self.callback_handler.on_evaluate(self.args		,     self.state		,     self.control		,     __A   )
												return metrics

							def       __A (  self:   Union[str, Any]		,     __A:   Optional[int]		,     __A:   int		,     __A:   List[Any]=None		,     __A:   str = "test"   )    ->				Union[str, Any]:
												_A								= self.get_test_dataloader(__A   )

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Prediction'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is None or self.compute_metrics is None:
																	return output

												_A								= self.post_process_function(__A		,     __A		,     output.predictions		,     '''predict'''   )
												_A								= self.compute_metrics(__A   )

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()   ):
																	if not key.startswith(f"""{metric_key_prefix}_"""   ):
																						_A								= metrics.pop(__A   )

												return PredictionOutput(predictions=predictions.predictions		,     label_ids=predictions.label_ids		,     metrics=__A   )



							def       __A (  self:   Tuple		,     __A:   Optional[Any]="./"   )    ->				List[str]:
												_A								= self.eval_dataset
												_A								= self.get_eval_dataloader(__A   )

												_A								= next(iter(__A   )   )

												# saving device - to make it consistent
												_A								= torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''   )

												# convert to tuple
												_A								= tuple(v.to(__A   ) for k, v in batch.items()   )

												logger.info('''Converting model to be onnx compatible'''   )
												from pytorch_quantization.nn import TensorQuantizer

												_A								= True

												_A								= self.model.to(__A   )

												model.eval()
												model.float()

												_A								= model.module if hasattr(__A		,     '''module'''   ) else model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args   )

												_A								= os.path.join(__A		,     '''model.onnx'''   )
												logger.info(f"""exporting model to {output_model_file}"""   )

												_A								= {0: '''batch_size''', 1: '''seq_len'''}

												torch.onnx.export(
												    __A		,     __A		,     __A		,     export_params=__A		,     opset_version=13		,     do_constant_folding=__A		,     input_names=['''input_ids''', '''attention_mask''', '''token_type_ids''']		,     output_names=['''output_start_logits''', '''output_end_logits''']		,     dynamic_axes={
												        '''input_ids''': axes,
												        '''attention_mask''': axes,
												        '''token_type_ids''': axes,
												        '''output_start_logits''': axes,
												        '''output_end_logits''': axes,
												    }		,     verbose=__A		,     )
												logger.info('''onnx export finished'''   )



