






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 						SCREAMING_SNAKE_CASE				( __lowerCamelCase  ,	unittest.TestCase					):





							"""simple docstring"""

							A_						  =					BertTokenizer
							A_						  =					BertTokenizerFast
							A_						  =					True
							A_						  =					True
							A_						  =					filter_non_english

							def       __A (  self:   int   )    ->				List[str]:
												super().setUp()

												_A								= [
												    "[UNK]",
												    "[CLS]",
												    "[SEP]",
												    "[PAD]",
												    "[MASK]",
												    "want",
												    "##want",
												    "##ed",
												    "wa",
												    "un",
												    "runn",
												    "##ing",
												    ",",
												    "low",
												    "lowest",
												]
												_A								= os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['''vocab_file''']   )
												with open(self.vocab_file		,     '''w'''		,     encoding='''utf-8'''   ) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )

							def       __A (  self:   Any		,     __A:   List[str]   )    ->				List[str]:
												_A								= "UNwant\u00E9d,running"
												_A								= "unwanted, running"
												return input_text, output_text

							def       __A (  self:   List[str]   )    ->				Union[str, Any]:
												_A								= self.tokenizer_class(self.vocab_file   )

												_A								= tokenizer.tokenize('''UNwant\u00E9d,running'''   )
												self.assertListEqual(a_		,     ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
												self.assertListEqual(tokenizer.convert_tokens_to_ids(a_   )		,     [9, 6, 7, 12, 10, 11]   )

							def       __A (  self:   Any   )    ->				List[str]:
												if not self.test_rust_tokenizer:
																	return

												_A								= self.get_tokenizer()
												_A								= self.get_rust_tokenizer()

												_A								= "UNwant\u00E9d,running"

												_A								= tokenizer.tokenize(a_   )
												_A								= rust_tokenizer.tokenize(a_   )
												self.assertListEqual(a_		,     a_   )

												_A								= tokenizer.encode(a_		,     add_special_tokens=a_   )
												_A								= rust_tokenizer.encode(a_		,     add_special_tokens=a_   )
												self.assertListEqual(a_		,     a_   )

												_A								= self.get_rust_tokenizer()
												_A								= tokenizer.encode(a_   )
												_A								= rust_tokenizer.encode(a_   )
												self.assertListEqual(a_		,     a_   )

												# With lower casing
												_A								= self.get_tokenizer(do_lower_case=a_   )
												_A								= self.get_rust_tokenizer(do_lower_case=a_   )

												_A								= "UNwant\u00E9d,running"

												_A								= tokenizer.tokenize(a_   )
												_A								= rust_tokenizer.tokenize(a_   )
												self.assertListEqual(a_		,     a_   )

												_A								= tokenizer.encode(a_		,     add_special_tokens=a_   )
												_A								= rust_tokenizer.encode(a_		,     add_special_tokens=a_   )
												self.assertListEqual(a_		,     a_   )

												_A								= self.get_rust_tokenizer()
												_A								= tokenizer.encode(a_   )
												_A								= rust_tokenizer.encode(a_   )
												self.assertListEqual(a_		,     a_   )

							def       __A (  self:   int   )    ->				int:
												_A								= BasicTokenizer()

												self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )		,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )

							def       __A (  self:   Dict   )    ->				List[str]:
												_A								= BasicTokenizer(do_lower_case=a_   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )		,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,     ['''hello''']   )

							def       __A (  self:   Tuple   )    ->				Optional[Any]:
												_A								= BasicTokenizer(do_lower_case=a_		,     strip_accents=a_   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,     ['''h\u00E9llo''']   )

							def       __A (  self:   Tuple   )    ->				str:
												_A								= BasicTokenizer(do_lower_case=a_		,     strip_accents=a_   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,     ['''hello''']   )

							def       __A (  self:   Dict   )    ->				Optional[int]:
												_A								= BasicTokenizer(do_lower_case=a_   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,     ['''hello''']   )

							def       __A (  self:   Optional[int]   )    ->				Any:
												_A								= BasicTokenizer(do_lower_case=a_   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )		,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )

							def       __A (  self:   Optional[int]   )    ->				List[Any]:
												_A								= BasicTokenizer(do_lower_case=a_		,     strip_accents=a_   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )

							def       __A (  self:   Any   )    ->				Optional[int]:
												_A								= BasicTokenizer(do_lower_case=a_		,     strip_accents=a_   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )

							def       __A (  self:   int   )    ->				Optional[Any]:
												_A								= BasicTokenizer(do_lower_case=a_		,     never_split=['''[UNK]''']   )

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )		,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )

							def       __A (  self:   Dict   )    ->				Tuple:
												_A								= BasicTokenizer()
												_A								= "a\n'll !!to?'d of, can't."
												_A								= ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
												self.assertListEqual(tokenizer.tokenize(a_   )		,     a_   )

							def       __A (  self:   int   )    ->				List[Any]:
												_A								= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

												_A								= {}
												for i, token in enumerate(a_   ):
																	_A								= i
												_A								= WordpieceTokenizer(vocab=a_		,     unk_token='''[UNK]'''   )

												self.assertListEqual(tokenizer.tokenize(''''''   )		,     []   )

												self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )		,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

												self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )		,     ['''[UNK]''', '''runn''', '''##ing''']   )

							def       __A (  self:   Optional[int]   )    ->				List[str]:
												self.assertTrue(_is_whitespace(''' '''   )   )
												self.assertTrue(_is_whitespace('''\t'''   )   )
												self.assertTrue(_is_whitespace('''\r'''   )   )
												self.assertTrue(_is_whitespace('''\n'''   )   )
												self.assertTrue(_is_whitespace('''\u00A0'''   )   )

												self.assertFalse(_is_whitespace('''A'''   )   )
												self.assertFalse(_is_whitespace('''-'''   )   )

							def       __A (  self:   Dict   )    ->				str:
												self.assertTrue(_is_control('''\u0005'''   )   )

												self.assertFalse(_is_control('''A'''   )   )
												self.assertFalse(_is_control(''' '''   )   )
												self.assertFalse(_is_control('''\t'''   )   )
												self.assertFalse(_is_control('''\r'''   )   )

							def       __A (  self:   Union[str, Any]   )    ->				Dict:
												self.assertTrue(_is_punctuation('''-'''   )   )
												self.assertTrue(_is_punctuation('''$'''   )   )
												self.assertTrue(_is_punctuation('''`'''   )   )
												self.assertTrue(_is_punctuation('''.'''   )   )

												self.assertFalse(_is_punctuation('''A'''   )   )
												self.assertFalse(_is_punctuation(''' '''   )   )

							def       __A (  self:   str   )    ->				List[Any]:
												_A								= self.get_tokenizer()
												_A								= self.get_rust_tokenizer()

												# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
												self.assertListEqual([tokenizer.tokenize(a_   ) for t in ['''Test''', '''\xad''', '''test''']]		,     [['''[UNK]'''], [], ['''[UNK]''']]   )

												self.assertListEqual(
												    [rust_tokenizer.tokenize(a_   ) for t in ['''Test''', '''\xad''', '''test''']]		,     [['''[UNK]'''], [], ['''[UNK]''']]   )

							@slow
							def       __A (  self:   Tuple   )    ->				Optional[Any]:
												_A								= self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

												_A								= tokenizer.encode('''sequence builders'''		,     add_special_tokens=a_   )
												_A								= tokenizer.encode('''multi-sequence build'''		,     add_special_tokens=a_   )

												_A								= tokenizer.build_inputs_with_special_tokens(a_   )
												_A								= tokenizer.build_inputs_with_special_tokens(a_		,     a_   )

												assert encoded_sentence == [1_01] + text + [1_02]
												assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

							def       __A (  self:   List[str]   )    ->				Optional[Any]:
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																						_A								= self.rust_tokenizer_class.from_pretrained(a_		,     **a_   )

																						_A								= f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						_A								= tokenizer_r.encode_plus(
																						    a_		,     return_attention_mask=a_		,     return_token_type_ids=a_		,     return_offsets_mapping=a_		,     add_special_tokens=a_		,     )

																						_A								= tokenizer_r.do_lower_case if hasattr(a_		,     '''do_lower_case'''   ) else False
																						_A								= (
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), "A"),
																						        ((1, 2), ","),
																						        ((3, 5), "na"),
																						        ((5, 6), "##ï"),
																						        ((6, 8), "##ve"),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), "Allen"),
																						        ((21, 23), "##NL"),
																						        ((23, 24), "##P"),
																						        ((25, 33), "sentence"),
																						        ((33, 34), "."),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), "a"),
																						        ((1, 2), ","),
																						        ((3, 8), "naive"),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), "allen"),
																						        ((21, 23), "##nl"),
																						        ((23, 24), "##p"),
																						        ((25, 33), "sentence"),
																						        ((33, 34), "."),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
																						self.assertEqual([e[0] for e in expected_results]		,     tokens['''offset_mapping''']   )



							def       __A (  self:   List[str]   )    ->				Union[str, Any]:
												_A								= ["的", "人", "有"]
												_A								= "".join(a_   )
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																						_A								= True
																						_A								= self.tokenizer_class.from_pretrained(a_		,     **a_   )
																						_A								= self.rust_tokenizer_class.from_pretrained(a_		,     **a_   )

																						_A								= tokenizer_p.encode(a_		,     add_special_tokens=a_   )
																						_A								= tokenizer_r.encode(a_		,     add_special_tokens=a_   )

																						_A								= tokenizer_r.convert_ids_to_tokens(a_   )
																						_A								= tokenizer_p.convert_ids_to_tokens(a_   )

																						# it is expected that each Chinese character is not preceded by "##"
																						self.assertListEqual(a_		,     a_   )
																						self.assertListEqual(a_		,     a_   )

																						_A								= False
																						_A								= self.rust_tokenizer_class.from_pretrained(a_		,     **a_   )
																						_A								= self.tokenizer_class.from_pretrained(a_		,     **a_   )

																						_A								= tokenizer_r.encode(a_		,     add_special_tokens=a_   )
																						_A								= tokenizer_p.encode(a_		,     add_special_tokens=a_   )

																						_A								= tokenizer_r.convert_ids_to_tokens(a_   )
																						_A								= tokenizer_p.convert_ids_to_tokens(a_   )

																						# it is expected that only the first Chinese character is not preceded by "##".
																						_A								= [
																						    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a_   )
																						]
																						self.assertListEqual(a_		,     a_   )
																						self.assertListEqual(a_		,     a_   )











import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__A      =				logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm
				import torch_xla.debug.metrics as met





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							def __init__(  self:   int		,     *__A:   str		,     __A:   List[Any]=None		,     __A:   Union[str, Any]=None		,     __A:   List[Any]=None		,     **__A:   int   )    ->				List[Any]:
												super().__init__(*__A		,     **__A   )
												_A								= eval_examples
												_A								= post_process_function
												_A								= quant_trainer_args
												_A								= 1_28  # default number of calibration samples

							def       __A (  self:   Union[str, Any]		,     __A:   List[Any]=None   )    ->				Optional[Any]:
												if calib_dataset is None and self.calib_dataset is None:
																	raise ValueError('''Trainer: calibration requires an calib_dataset.'''   )
												_A								= calib_dataset if calib_dataset is not None else self.calib_dataset

												_A								= self._remove_unused_columns(__A		,     description='''Calibration'''   )

												return DataLoader(
												    __A		,     batch_size=self.args.eval_batch_size		,     collate_fn=self.data_collator		,     drop_last=self.args.dataloader_drop_last		,     num_workers=self.args.dataloader_num_workers		,     pin_memory=self.args.dataloader_pin_memory		,     shuffle=__A		,     )

							def       __A (  self:   List[Any]		,     __A:   Any=None   )    ->				Optional[int]:
												_A								= self.train_dataset if calib_dataset is None else calib_dataset
												_A								= self.get_calib_dataloader(__A   )

												_A								= self.model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args		,     calib=__A   )
												model.eval()
												quant_trainer.enable_calibration(__A   )

												logger.info('''***** Running calibration *****'''   )
												logger.info(f"""  Num examples = {self.calib_num}"""   )
												logger.info(f"""  Batch size = {calib_dataloader.batch_size}"""   )

												for step, inputs in enumerate(__A   ):
																	# Prediction step
																	_A							,_A							,_A								= self.prediction_step(__A		,     __A		,     prediction_loss_only=__A   )
																	if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
																						break

												quant_trainer.finish_calibration(__A		,     self.quant_trainer_args   )
												_A								= model

							def       __A (  self:   Any		,     __A:   Dict=None		,     __A:   Tuple=None		,     __A:   List[Any]=None		,     __A:   str = "eval"   )    ->				int:
												_A								= self.eval_dataset if eval_dataset is None else eval_dataset
												_A								= self.get_eval_dataloader(__A   )
												_A								= self.eval_examples if eval_examples is None else eval_examples

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Evaluation'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is not None and self.compute_metrics is not None:
																	_A								= self.post_process_function(__A		,     __A		,     output.predictions   )
																	_A								= self.compute_metrics(__A   )

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()   ):
																						if not key.startswith(f"""{metric_key_prefix}_"""   ):
																											_A								= metrics.pop(__A   )

																	self.log(__A   )
												else:
																	_A								= {}

												if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report()   )

												_A								= self.callback_handler.on_evaluate(self.args		,     self.state		,     self.control		,     __A   )
												return metrics

							def       __A (  self:   Union[str, Any]		,     __A:   Optional[int]		,     __A:   int		,     __A:   List[Any]=None		,     __A:   str = "test"   )    ->				Union[str, Any]:
												_A								= self.get_test_dataloader(__A   )

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Prediction'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is None or self.compute_metrics is None:
																	return output

												_A								= self.post_process_function(__A		,     __A		,     output.predictions		,     '''predict'''   )
												_A								= self.compute_metrics(__A   )

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()   ):
																	if not key.startswith(f"""{metric_key_prefix}_"""   ):
																						_A								= metrics.pop(__A   )

												return PredictionOutput(predictions=predictions.predictions		,     label_ids=predictions.label_ids		,     metrics=__A   )



							def       __A (  self:   Tuple		,     __A:   Optional[Any]="./"   )    ->				List[str]:
												_A								= self.eval_dataset
												_A								= self.get_eval_dataloader(__A   )

												_A								= next(iter(__A   )   )

												# saving device - to make it consistent
												_A								= torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''   )

												# convert to tuple
												_A								= tuple(v.to(__A   ) for k, v in batch.items()   )

												logger.info('''Converting model to be onnx compatible'''   )
												from pytorch_quantization.nn import TensorQuantizer

												_A								= True

												_A								= self.model.to(__A   )

												model.eval()
												model.float()

												_A								= model.module if hasattr(__A		,     '''module'''   ) else model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args   )

												_A								= os.path.join(__A		,     '''model.onnx'''   )
												logger.info(f"""exporting model to {output_model_file}"""   )

												_A								= {0: '''batch_size''', 1: '''seq_len'''}

												torch.onnx.export(
												    __A		,     __A		,     __A		,     export_params=__A		,     opset_version=13		,     do_constant_folding=__A		,     input_names=['''input_ids''', '''attention_mask''', '''token_type_ids''']		,     output_names=['''output_start_logits''', '''output_end_logits''']		,     dynamic_axes={
												        '''input_ids''': axes,
												        '''attention_mask''': axes,
												        '''token_type_ids''': axes,
												        '''output_start_logits''': axes,
												        '''output_end_logits''': axes,
												    }		,     verbose=__A		,     )
												logger.info('''onnx export finished'''   )



