





'''simple docstring'''



import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
	import torch_xla.core.xla_model as xm
	import torch_xla.debug.metrics as met






class 							__lowercase       (      _UpperCAmelCase						):







	'''simple docstring'''

	def __init__(self     ,*_lowerCamelCase     ,_lowerCamelCase=None     ,_lowerCamelCase=None     ,**_lowerCamelCase )							->							Optional[int]:







				'''simple docstring'''





				super().__init__(*lowercase__     ,**lowercase__ )
				__lowercase										=							eval_examples
				__lowercase										=							post_process_function


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase = None     ,_lowerCamelCase=None     ,_lowerCamelCase = None     ,_lowerCamelCase = "eval"     ,**_lowerCamelCase     ,)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							gen_kwargs.copy()
				__lowercase										=							(
				    gen_kwargs["max_length"] if gen_kwargs.get('''max_length''' ) is not None else self.args.generation_max_length
				)
				__lowercase										=							(
				    gen_kwargs["num_beams"] if gen_kwargs.get('''num_beams''' ) is not None else self.args.generation_num_beams
				)
				__lowercase										=							gen_kwargs

				__lowercase										=							self.eval_dataset if eval_dataset is None else eval_dataset
				__lowercase										=							self.get_eval_dataloader(lowercase__ )
				__lowercase										=							self.eval_examples if eval_examples is None else eval_examples

				# Temporarily disable metric computation, we will do it in the loop here.
				__lowercase										=							self.compute_metrics
				__lowercase										=							None
				__lowercase										=							time.time()
				__lowercase										=							self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
				try:
							__lowercase										=							eval_loop(
							    lowercase__     ,description='''Evaluation'''     ,prediction_loss_only=True if compute_metrics is None else None     ,ignore_keys=lowercase__     ,metric_key_prefix=lowercase__     ,)
				finally:
							__lowercase										=							compute_metrics
				__lowercase										=							self.args.eval_batch_size * self.args.world_size
				if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
							start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
				output.metrics.update(
				    speed_metrics(
				        lowercase__     ,lowercase__     ,num_samples=output.num_samples     ,num_steps=math.ceil(output.num_samples / total_batch_size )     ,) )

				if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
							# Only the main node write the results by default
							__lowercase										=							self.post_process_function(lowercase__     ,lowercase__     ,lowercase__ )
							__lowercase										=							self.compute_metrics(lowercase__ )

							# Prefix all keys with metric_key_prefix + '_'
							for key in list(metrics.keys() ):
										if not key.startswith(f"{metric_key_prefix}_" ):
													__lowercase										=							metrics.pop(lowercase__ )

							metrics.update(output.metrics )
				else:
							__lowercase										=							output.metrics

				if self.args.should_log:
							# Only the main node log the results by default
							self.log(lowercase__ )

				if self.args.tpu_metrics_debug or self.args.debug:
							# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
							xm.master_print(met.metrics_report() )

				__lowercase										=							self.callback_handler.on_evaluate(self.args     ,self.state     ,self.control     ,lowercase__ )
				return metrics


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase=None     ,_lowerCamelCase = "test"     ,**_lowerCamelCase )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							gen_kwargs.copy()

				__lowercase										=							self.get_test_dataloader(lowercase__ )

				# Temporarily disable metric computation, we will do it in the loop here.
				__lowercase										=							self.compute_metrics
				__lowercase										=							None
				__lowercase										=							time.time()
				__lowercase										=							self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
				try:
							__lowercase										=							eval_loop(
							    lowercase__     ,description='''Prediction'''     ,prediction_loss_only=True if compute_metrics is None else None     ,ignore_keys=lowercase__     ,metric_key_prefix=lowercase__     ,)
				finally:
							__lowercase										=							compute_metrics

				__lowercase										=							self.args.eval_batch_size * self.args.world_size
				if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
							start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
				output.metrics.update(
				    speed_metrics(
				        lowercase__     ,lowercase__     ,num_samples=output.num_samples     ,num_steps=math.ceil(output.num_samples / total_batch_size )     ,) )
				if self.post_process_function is None or self.compute_metrics is None:
							return output

				__lowercase										=							self.post_process_function(lowercase__     ,lowercase__     ,lowercase__     ,'''predict''' )
				__lowercase										=							self.compute_metrics(lowercase__ )

				# Prefix all keys with metric_key_prefix + '_'
				for key in list(metrics.keys() ):
							if not key.startswith(f"{metric_key_prefix}_" ):
										__lowercase										=							metrics.pop(lowercase__ )
				metrics.update(output.metrics )
				return PredictionOutput(predictions=predictions.predictions     ,label_ids=predictions.label_ids     ,metrics=lowercase__ )






'''simple docstring'''



import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def __init__(self     ,_lowerCamelCase = "▁"     ,_lowerCamelCase = True     ,_lowerCamelCase = "<unk>"     ,_lowerCamelCase = "</s>"     ,_lowerCamelCase = "<pad>"     ,)							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							{
				    '''pad''': {'''id''': 0, '''token''': pad_token},
				    '''eos''': {'''id''': 1, '''token''': eos_token},
				    '''unk''': {'''id''': 2, '''token''': unk_token},
				}

				__lowercase										=							[None] * len(self.special_tokens )
				for token_dict in self.special_tokens.values():
							__lowercase										=							token_dict['''token''']

				__lowercase										=							Tokenizer(Unigram() )

				__lowercase										=							normalizers.Sequence(
				    [
				        normalizers.Nmt(),
				        normalizers.NFKC(),
				        normalizers.Replace(Regex(''' {2,}''' )     ,''' ''' ),
				        normalizers.Lowercase(),
				    ] )
				__lowercase										=							pre_tokenizers.Sequence(
				    [
				        pre_tokenizers.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase ),
				        pre_tokenizers.Digits(individual_digits=_lowerCamelCase ),
				        pre_tokenizers.Punctuation(),
				    ] )
				__lowercase										=							decoders.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase )

				__lowercase										=							TemplateProcessing(
				    single=f"$A {self.special_tokens['eos']['token']}"     ,special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]     ,)

				__lowercase										=							{
				    '''model''': '''SentencePieceUnigram''',
				    '''replacement''': replacement,
				    '''add_prefix_space''': add_prefix_space,
				}

				super().__init__(_lowerCamelCase     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				if isinstance(_lowerCamelCase     ,_lowerCamelCase ):
							__lowercase										=							[files]
				self._tokenizer.train(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							List[str]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				self._tokenizer.train_from_iterator(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							json.loads(self._tokenizer.to_str() )

				__lowercase										=							self.special_tokens['''unk''']['''id''']

				__lowercase										=							Tokenizer.from_str(json.dumps(_lowerCamelCase ) )





