





import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from seqaseq_trainer import SeqaSeqTrainer
from seqaseq_training_args import SeqaSeqTrainingArguments

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSeqaSeqLM,
    AutoTokenizer,
    HfArgumentParser,
    MBartTokenizer,
    MBartTokenizerFast,
    set_seed,
)
from transformers.trainer_utils import EvaluationStrategy, is_main_process
from transformers.training_args import ParallelMode
from utils import (
    SeqaSeqDataCollator,
    SeqaSeqDataset,
    assert_all_frozen,
    build_compute_metrics_fn,
    check_output_dir,
    freeze_embeds,
    freeze_params,
    lmap,
    save_json,
    use_task_specific_params,
    write_txt_file,
)


__lowerCamelCase :  List[Any]										=  logging.getLogger(__name__)



@dataclass
class       A__     :
      _UpperCAmelCase			:int							=     field(
          metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}    )
      _UpperCAmelCase			:str							=     field(
          default=__snake_case ,     metadata={'help': 'Pretrained config name or path if not the same as model_name'}    )
      _UpperCAmelCase			:Dict							=     field(
          default=__snake_case ,     metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}    )
      _UpperCAmelCase			:str							=     field(
          default=__snake_case ,     metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'} ,     )
      _UpperCAmelCase			:Optional[Any]							=     field(default=__snake_case ,     metadata={'help': 'Whether tp freeze the encoder.'}    )
      _UpperCAmelCase			:Any							=     field(default=__snake_case ,     metadata={'help': 'Whether  to freeze the embeddings.'}    )



@dataclass
class       A__     :
      _UpperCAmelCase			:List[str]							=     field(
          metadata={'help': 'The input data dir. Should contain the .tsv files (or other data files) for the task.'}    )
      _UpperCAmelCase			:int							=     field(
          default='summarization' ,     metadata={'help': 'Task name, summarization (or summarization_{dataset} for pegasus) or translation'} ,     )
      _UpperCAmelCase			:Optional[int]							=     field(
          default=1_0_2_4 ,     metadata={
              'help': (
                  'The maximum total input sequence length after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )
          } ,     )
      _UpperCAmelCase			:Optional[int]							=     field(
          default=1_2_8 ,     metadata={
              'help': (
                  'The maximum total sequence length for target text after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )
          } ,     )
      _UpperCAmelCase			:str							=     field(
          default=1_4_2 ,     metadata={
              'help': (
                  'The maximum total sequence length for validation target text after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded. '
                  'This argument is also used to override the ``max_length`` param of ``model.generate``, which is used '
                  'during ``evaluate`` and ``predict``.'
              )
          } ,     )
      _UpperCAmelCase			:int							=     field(
          default=1_4_2 ,     metadata={
              'help': (
                  'The maximum total sequence length for test target text after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )
          } ,     )
      _UpperCAmelCase			:Optional[Any]							=     field(default=-1 ,     metadata={'help': '# training examples. -1 means use all.'}    )
      _UpperCAmelCase			:Tuple							=     field(default=-1 ,     metadata={'help': '# validation examples. -1 means use all.'}    )
      _UpperCAmelCase			:List[Any]							=     field(default=-1 ,     metadata={'help': '# test examples. -1 means use all.'}    )
      _UpperCAmelCase			:str							=     field(default=__snake_case ,     metadata={'help': 'Source language id for translation.'}    )
      _UpperCAmelCase			:str							=     field(default=__snake_case ,     metadata={'help': 'Target language id for translation.'}    )
      _UpperCAmelCase			:Dict							=     field(default=__snake_case ,     metadata={'help': '# num_beams to use for evaluation.'}    )
      _UpperCAmelCase			:Tuple							=     field(
          default=__snake_case ,     metadata={'help': 'If only pad tokens should be ignored. This assumes that `config.pad_token_id` is defined.'} ,     )



def 	A_       (  _lowerCAmelCase							,     _lowerCAmelCase							,     _lowerCAmelCase  )		->				Tuple:
 logger.info(F"""***** {split} metrics *****"""  )
 for key in sorted(metrics.keys()  ):
  logger.info(F"""  {key} = {metrics[key]}"""  )
 save_json(_UpperCamelCase							,     os.path.join(_UpperCamelCase							,     F"""{split}_results.json"""  )  )



def 	A_       (  )		->				Any:
 UpperCamelCase			:      Dict 			=     HfArgumentParser((ModelArguments, DataTrainingArguments, SeqaSeqTrainingArguments)  )

 if len(sys.argv  ) == 2 and sys.argv[1].endswith(".json"  ):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
  UpperCamelCase			,					UpperCamelCase			,					UpperCamelCase			:      int 			=     parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]  )  )
 else:
  UpperCamelCase			,					UpperCamelCase			,					UpperCamelCase			:      Optional[Any] 			=     parser.parse_args_into_dataclasses()

 check_output_dir(_UpperCamelCase  )

 # Setup logging
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"							,     datefmt="%m/%d/%Y %H:%M:%S"							,     level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN							,     )
 logger.warning(
     "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"							,     training_args.local_rank							,     training_args.device							,     training_args.n_gpu							,     bool(training_args.parallel_mode == ParallelMode.DISTRIBUTED  )							,     training_args.fpaa							,     )
 transformers.utils.logging.enable_default_handler()
 transformers.utils.logging.enable_explicit_format()
 # Set the verbosity to info of the Transformers logger (on main process only):
 if is_main_process(training_args.local_rank  ):
  transformers.utils.logging.set_verbosity_info()
 logger.info("Training/evaluation parameters %s"							,     _UpperCamelCase  )

 # Set seed
 set_seed(training_args.seed  )

 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 UpperCamelCase			:      Union[str, Any] 			=     AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path							,     cache_dir=model_args.cache_dir							,     )

 UpperCamelCase			:      Optional[Any] 			=     ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
 for p in extra_model_params:
  if getattr(_UpperCamelCase							,     _UpperCamelCase							,     _UpperCamelCase  ):
   assert hasattr(_UpperCamelCase							,     _UpperCamelCase  ), F"""({config.__class__.__name__}) doesn't have a `{p}` attribute"""
   setattr(_UpperCamelCase							,     _UpperCamelCase							,     getattr(_UpperCamelCase							,     _UpperCamelCase  )  )

 UpperCamelCase			:      Union[str, Any] 			=     AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path							,     cache_dir=model_args.cache_dir							,     )
 UpperCamelCase			:      Optional[int] 			=     AutoModelForSeqaSeqLM.from_pretrained(
     model_args.model_name_or_path							,     from_tf=".ckpt" in model_args.model_name_or_path							,     config=_UpperCamelCase							,     cache_dir=model_args.cache_dir							,     )

 # use task specific params
 use_task_specific_params(_UpperCamelCase							,     data_args.task  )

 # set num_beams for evaluation
 if data_args.eval_beams is None:
  UpperCamelCase			:      str 			=     model.config.num_beams

 # set decoder_start_token_id for MBart
 if model.config.decoder_start_token_id is None and isinstance(_UpperCamelCase							,     (MBartTokenizer, MBartTokenizerFast)  ):
  assert (
      data_args.tgt_lang is not None and data_args.src_lang is not None
  ), "mBart requires --tgt_lang and --src_lang"
  if isinstance(_UpperCamelCase							,     _UpperCamelCase  ):
   UpperCamelCase			:      Tuple 			=     tokenizer.lang_code_to_id[data_args.tgt_lang]
  else:
   UpperCamelCase			:      Optional[int] 			=     tokenizer.convert_tokens_to_ids(data_args.tgt_lang  )

 if model_args.freeze_embeds:
  freeze_embeds(_UpperCamelCase  )
 if model_args.freeze_encoder:
  freeze_params(model.get_encoder()  )
  assert_all_frozen(model.get_encoder()  )

 UpperCamelCase			:      int 			=     SeqaSeqDataset

 # Get datasets
 UpperCamelCase			:      int 			=     (
     dataset_class(
         _UpperCamelCase							,     type_path="train"							,     data_dir=data_args.data_dir							,     n_obs=data_args.n_train							,     max_target_length=data_args.max_target_length							,     max_source_length=data_args.max_source_length							,     prefix=model.config.prefix or ""							,     )
     if training_args.do_train
     else None
 )
 UpperCamelCase			:      int 			=     (
     dataset_class(
         _UpperCamelCase							,     type_path="val"							,     data_dir=data_args.data_dir							,     n_obs=data_args.n_val							,     max_target_length=data_args.val_max_target_length							,     max_source_length=data_args.max_source_length							,     prefix=model.config.prefix or ""							,     )
     if training_args.do_eval or training_args.evaluation_strategy != EvaluationStrategy.NO
     else None
 )
 UpperCamelCase			:      str 			=     (
     dataset_class(
         _UpperCamelCase							,     type_path="test"							,     data_dir=data_args.data_dir							,     n_obs=data_args.n_test							,     max_target_length=data_args.test_max_target_length							,     max_source_length=data_args.max_source_length							,     prefix=model.config.prefix or ""							,     )
     if training_args.do_predict
     else None
 )

 # Initialize our Trainer
 UpperCamelCase			:      Tuple 			=     (
     build_compute_metrics_fn(data_args.task							,     _UpperCamelCase  ) if training_args.predict_with_generate else None
 )
 UpperCamelCase			:      Any 			=     SeqaSeqTrainer(
     model=_UpperCamelCase							,     args=_UpperCamelCase							,     data_args=_UpperCamelCase							,     train_dataset=_UpperCamelCase							,     eval_dataset=_UpperCamelCase							,     data_collator=SeqaSeqDataCollator(
         _UpperCamelCase							,     _UpperCamelCase							,     model.config.decoder_start_token_id							,     training_args.tpu_num_cores  )							,     compute_metrics=_UpperCamelCase							,     tokenizer=_UpperCamelCase							,     )

 UpperCamelCase			:      Union[str, Any] 			=     {}
 # Training
 if training_args.do_train:
  logger.info("*** Train ***"  )

  UpperCamelCase			:      Optional[Any] 			=     trainer.train(
      model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path  ) else None  )
  UpperCamelCase			:      Optional[int] 			=     train_result.metrics
  UpperCamelCase			:      List[Any] 			=     data_args.n_train

  trainer.save_model()  # this also saves the tokenizer

  if trainer.is_world_process_zero():
   handle_metrics("train"							,     _UpperCamelCase							,     training_args.output_dir  )
   all_metrics.update(_UpperCamelCase  )

   # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
   trainer.state.save_to_json(os.path.join(training_args.output_dir							,     "trainer_state.json"  )  )

   # For convenience, we also re-save the tokenizer to the same directory,
   # so that you can share your model easily on huggingface.co/models =)
   tokenizer.save_pretrained(training_args.output_dir  )

    # Evaluation
 if training_args.do_eval:
  logger.info("*** Evaluate ***"  )

  UpperCamelCase			:      Optional[int] 			=     trainer.evaluate(metric_key_prefix="val"  )
  UpperCamelCase			:      int 			=     data_args.n_val
  UpperCamelCase			:      Dict 			=     round(metrics["val_loss"]							,     4  )

  if trainer.is_world_process_zero():
   handle_metrics("val"							,     _UpperCamelCase							,     training_args.output_dir  )
   all_metrics.update(_UpperCamelCase  )

 if training_args.do_predict:
  logger.info("*** Predict ***"  )

  UpperCamelCase			:      Any 			=     trainer.predict(test_dataset=_UpperCamelCase							,     metric_key_prefix="test"  )
  UpperCamelCase			:      Optional[int] 			=     test_output.metrics
  UpperCamelCase			:      Optional[Any] 			=     data_args.n_test

  if trainer.is_world_process_zero():
   UpperCamelCase			:      Union[str, Any] 			=     round(metrics["test_loss"]							,     4  )
   handle_metrics("test"							,     _UpperCamelCase							,     training_args.output_dir  )
   all_metrics.update(_UpperCamelCase  )

   if training_args.predict_with_generate:
    UpperCamelCase			:      int 			=     tokenizer.batch_decode(
        test_output.predictions							,     skip_special_tokens=_UpperCamelCase							,     clean_up_tokenization_spaces=_UpperCamelCase  )
    UpperCamelCase			:      Optional[Any] 			=     lmap(str.strip							,     _UpperCamelCase  )
    write_txt_file(_UpperCamelCase							,     os.path.join(training_args.output_dir							,     "test_generations.txt"  )  )

 if trainer.is_world_process_zero():
  save_json(_UpperCamelCase							,     os.path.join(training_args.output_dir							,     "all_results.json"  )  )

 return all_metrics



def 	A_       (  _lowerCAmelCase  )		->				Optional[Any]:
 main()


if __name__ == "__main__":
 main()







import math

import tensorflow as tf
from packaging import version



def 	A_       (  _lowerCAmelCase  )		->				Any:
 UpperCamelCase			:      List[Any] 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      Any 			=     0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0  )							,     x.dtype  )  ))

 return x * cdf



def 	A_       (  _lowerCAmelCase  )		->				Dict:
 UpperCamelCase			:      Union[str, Any] 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      List[Any] 			=     tf.cast(math.pi							,     x.dtype  )
 UpperCamelCase			:      Optional[Any] 			=     tf.cast(0.044_715							,     x.dtype  )
 UpperCamelCase			:      int 			=     0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi  ) * (x + coeff * tf.pow(_lowerCAmelCase							,     3  ))  ))

 return x * cdf



def 	A_       (  _lowerCAmelCase  )		->				List[Any]:
 UpperCamelCase			:      str 			=     tf.convert_to_tensor(_lowerCAmelCase  )

 return x * tf.tanh(tf.math.softplus(_lowerCAmelCase  )  )



def 	A_       (  _lowerCAmelCase  )		->				List[Any]:
 UpperCamelCase			:      Tuple 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      List[Any] 			=     tf.cast(0.044_715							,     x.dtype  )
 UpperCamelCase			:      Optional[Any] 			=     tf.cast(0.7_978_845_608							,     x.dtype  )

 return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)  ))



def 	A_       (  _lowerCAmelCase  )		->				Optional[Any]:
 UpperCamelCase			:      Any 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      List[Any] 			=     tf.cast(1.702							,     x.dtype  )
 return x * tf.math.sigmoid(coeff * x  )



def 	A_       (  _lowerCAmelCase  )		->				List[Any]:
 return tf.clip_by_value(_gelu(_lowerCAmelCase  )							,     -10							,     10  )



def 	A_       (  _lowerCAmelCase							,     _lowerCAmelCase=-1  )		->				str:
 UpperCamelCase			,					UpperCamelCase			:      List[Any] 			=     tf.split(_lowerCAmelCase							,     2							,     axis=_lowerCAmelCase  )
 return a * tf.math.sigmoid(_lowerCAmelCase  )


if version.parse(tf.version.VERSION) >= version.parse("""2.4"""):



    def 	A_       (  _lowerCAmelCase  )		->				Any:
     return tf.keras.activations.gelu(_lowerCAmelCase							,     approximate=_lowerCAmelCase  )

    __lowerCamelCase :  Optional[int]										=  tf.keras.activations.gelu


    __lowerCamelCase :  int										=  approximate_gelu_wrap
else:
    __lowerCamelCase :  List[Any]										=  _gelu
    __lowerCamelCase :  Optional[Any]										=  _gelu_new


__lowerCamelCase :  Any										=  {
    """gelu""": gelu,
    """gelu_10""": gelu_aa,
    """gelu_fast""": gelu_fast,
    """gelu_new""": gelu_new,
    """glu""": glu,
    """mish""": mish,
    """quick_gelu""": quick_gelu,
    """relu""": tf.keras.activations.relu,
    """sigmoid""": tf.keras.activations.sigmoid,
    """silu""": tf.keras.activations.swish,
    """swish""": tf.keras.activations.swish,
    """tanh""": tf.keras.activations.tanh,
}



def 	A_       (  _lowerCAmelCase  )		->				Optional[Any]:
 if activation_string in ACTaFN:
  return ACTaFN[activation_string]
 else:
  raise KeyError(F"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()  )}"""  )
