




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_snake_case            = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_snake_case            = {
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

_snake_case            = {
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

_snake_case            = {
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}







class 		lowercase							(     UpperCamelCase__	):
     _a											=		VOCAB_FILES_NAMES
     _a											=		PRETRAINED_VOCAB_FILES_MAP
     _a											=		PRETRAINED_INIT_CONFIGURATION
     _a											=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _a											=		ElectraTokenizer

     def __init__(						self	,  _a=None	,  _a=None	,  _a=True	,  _a="[UNK]"	,  _a="[SEP]"	,  _a="[PAD]"	,  _a="[CLS]"	,  _a="[MASK]"	,  _a=True	,  _a=None	,  **_a	,  )		->							int:
           super().__init__(
               _a	,  tokenizer_file=_a	,  do_lower_case=_a	,  unk_token=_a	,  sep_token=_a	,  pad_token=_a	,  cls_token=_a	,  mask_token=_a	,  tokenize_chinese_chars=_a	,  strip_accents=_a	,  **_a	,  )

           _A							:  int    =		json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
           if (
               normalizer_state.get("""lowercase"""	,  _a       ) != do_lower_case
               or normalizer_state.get("""strip_accents"""	,  _a       ) != strip_accents
               or normalizer_state.get("""handle_chinese_chars"""	,  _a       ) != tokenize_chinese_chars
           ):
                 _A							:  Dict    =		getattr(_a	,  normalizer_state.pop("""type"""       )       )
                 _A							:  Tuple    =		do_lower_case
                 _A							:  str    =		strip_accents
                 _A							:  int    =		tokenize_chinese_chars
                 _A							:  List[Any]    =		normalizer_class(**_a       )

           _A							:  Optional[int]    =		do_lower_case

     def       a__			(						self	,  _a	,  _a=None       )		->							str:
           _A							:  Optional[int]    =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output

     def       a__			(						self	,  _a	,  _a = None       )		->							List[int]:
           _A							:  str    =		[self.sep_token_id]
           _A							:  Any    =		[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep       ) * [0]
           return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]



     def       a__			(						self	,  _a	,  _a = None       )		->							Tuple[str]:
           _A							:  str    =		self._tokenizer.model.save(_a	,  name=_a       )
           return tuple(_a       )







from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
  from fairscale.optim import OSS


_snake_case            = logging.get_logger(__name__)

_snake_case            = {
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    "constant": get_constant_schedule,
    "constant_w_warmup": get_constant_schedule_with_warmup,
}







class 		lowercase							(     UpperCamelCase__	):

     def __init__(						self	,  _a=None	,  _a=None	,  *_a	,  **_a       )		->							Optional[int]:
           super().__init__(*_a	,  **_a       )

           if config is None:
                 assert isinstance(self.model	,  _a       ), (
                     "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
                     F''' {self.model.__class__}'''
                 )
                 _A							:  Optional[Any]    =		self.model.config
           else:
                 _A							:  int    =		config

           _A							:  Optional[Any]    =		data_args
           _A							:  int    =		self.config.tgt_vocab_size if isinstance(self.config	,  _a       ) else self.config.vocab_size

           if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
                 assert self.config.pad_token_id is not None, (
                     "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
                     " calculation or doing label smoothing."
                 )

           if self.config.pad_token_id is None and self.config.eos_token_id is not None:
                 logger.warning(
                     F'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
                     """ padding.."""       )

           if self.args.label_smoothing == 0:
                 _A							:  Optional[Any]    =		torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id       )
           else:
                 # dynamically import label_smoothed_nll_loss
                 from utils import label_smoothed_nll_loss

                 _A							:  Union[str, Any]    =		label_smoothed_nll_loss

     def       a__			(						self	,  _a       )		->							int:
           if self.optimizer is None:
                 _A							:  List[str]    =		["""bias""", """LayerNorm.weight"""]
                 _A							:  str    =		[
                     {
                         """params""": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay       )],
                         """weight_decay""": self.args.weight_decay,
                     },
                     {
                         """params""": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay       )],
                         """weight_decay""": 0.0,
                     },
                 ]
                 _A							:  Optional[Any]    =		Adafactor if self.args.adafactor else AdamW
                 if self.args.adafactor:
                       _A							:  Dict    =		Adafactor
                       _A							:  int    =		{"""scale_parameter""": False, """relative_step""": False}
                 else:
                       _A							:  int    =		AdamW
                       _A							:  Any    =		{
                           """betas""": (self.args.adam_betaa, self.args.adam_betaa),
                           """eps""": self.args.adam_epsilon,
                       }
                 _A							:  List[str]    =		self.args.learning_rate
                 if self.sharded_ddp:
                       _A							:  List[str]    =		OSS(
                           params=_a	,  optim=_a	,  **_a	,  )
                 else:
                       _A							:  Tuple    =		optimizer_cls(_a	,  **_a       )

           if self.lr_scheduler is None:
                 _A							:  Union[str, Any]    =		self._get_lr_scheduler(_a       )
           else:  # ignoring --lr_scheduler
                 logger.warning("""scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored."""       )

     def       a__			(						self	,  _a       )		->							Dict:
           _A							:  List[Any]    =		arg_to_scheduler[self.args.lr_scheduler]
           if self.args.lr_scheduler == "constant":
                 _A							:  Optional[Any]    =		schedule_func(self.optimizer       )
           elif self.args.lr_scheduler == "constant_w_warmup":
                 _A							:  Optional[int]    =		schedule_func(self.optimizer	,  num_warmup_steps=self.args.warmup_steps       )
           else:
                 _A							:  List[Any]    =		schedule_func(
                     self.optimizer	,  num_warmup_steps=self.args.warmup_steps	,  num_training_steps=_a       )
           return scheduler

     def       a__			(						self       )		->							Optional[torch.utils.data.Sampler]:
           if isinstance(self.train_dataset	,  torch.utils.data.IterableDataset       ):
                 return None
           elif is_torch_tpu_available():
                 return get_tpu_sampler(self.train_dataset       )
           else:
                 if self.args.sortish_sampler:
                       self.train_dataset.make_sortish_sampler(
                           self.args.per_device_train_batch_size	,  distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)	,  )

                 return (
                     RandomSampler(self.train_dataset       )
                     if self.args.local_rank == -1
                     else DistributedSampler(self.train_dataset       )
                 )

     def       a__			(						self	,  _a	,  _a	,  _a       )		->							List[str]:
           if self.args.label_smoothing == 0:
                 if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
                       # force training to ignore pad token
                       _A							:  List[str]    =		model(**_a	,  use_cache=_a       )[0]
                       _A							:  Dict    =		self.loss_fn(logits.view(-1	,  logits.shape[-1]       )	,  labels.view(-1       )       )
                 else:
                       # compute usual loss via models
                       _A    ,     _A							:  str    =		model(**_a	,  labels=_a	,  use_cache=_a       )[:2]
           else:
                 # compute label smoothed loss
                 _A							:  Any    =		model(**_a	,  use_cache=_a       )[0]
                 _A							:  Union[str, Any]    =		torch.nn.functional.log_softmax(_a	,  dim=-1       )
                 _A    ,     _A							:  List[str]    =		self.loss_fn(_a	,  _a	,  self.args.label_smoothing	,  ignore_index=self.config.pad_token_id       )
           return loss, logits

     def       a__			(						self	,  _a	,  _a       )		->							List[Any]:
           _A							:  Optional[int]    =		inputs.pop("""labels"""       )
           _A    ,     _A							:  Dict    =		self._compute_loss(_a	,  _a	,  _a       )
           return loss

     def       a__			(						self	,  _a	,  _a	,  _a	,  _a = None	,  )		->							Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
           _A							:  int    =		self._prepare_inputs(_a       )

           _A							:  Dict    =		{
               """max_length""": self.data_args.val_max_target_length
               if self.data_args is not None
               else self.config.max_length,
               """num_beams""": self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
           }

           if self.args.predict_with_generate and not self.args.prediction_loss_only:
                 _A							:  List[str]    =		self.model.generate(
                     inputs["""input_ids"""]	,  attention_mask=inputs["""attention_mask"""]	,  **_a	,  )
                 # in case the batch is shorter than max length, the output should be padded
                 if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
                       _A							:  str    =		self._pad_tensors_to_max_len(_a	,  gen_kwargs["""max_length"""]       )

           _A							:  Any    =		inputs.pop("""labels"""       )
           with torch.no_grad():
                 # compute loss on predict data
                 _A    ,     _A							:  Tuple    =		self._compute_loss(_a	,  _a	,  _a       )

           _A							:  Any    =		loss.mean().detach()
           if self.args.prediction_loss_only:
                 return (loss, None, None)

           _A							:  List[Any]    =		generated_tokens if self.args.predict_with_generate else logits

           if labels.shape[-1] < gen_kwargs["max_length"]:
                 _A							:  Any    =		self._pad_tensors_to_max_len(_a	,  gen_kwargs["""max_length"""]       )

           return (loss, logits, labels)



     def       a__			(						self	,  _a	,  _a       )		->							Union[str, Any]:
           # If PAD token is not defined at least EOS token has to be defined
           _A							:  Optional[Any]    =		self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

           if pad_token_id is None:
                 raise ValueError(
                     """Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be"""
                     F''' padded to `max_length`={max_length}'''       )

           _A							:  Dict    =		pad_token_id * torch.ones(
               (tensor.shape[0], max_length)	,  dtype=tensor.dtype	,  device=tensor.device       )
           _A							:  Dict    =		tensor
           return padded_tensor

