



import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


SCREAMING_SNAKE_CASE__						 =      logging.getLogger(__name__)


class      __lowerCAmelCase     (		SCREAMING_SNAKE_CASE_      ):
					"""simple docstring"""






					A__				:		Optional[int]			       =    "sequence-classification"



					def __init__(     self					:       Any				,					_snake_case					:       List[Any]					):






								"""simple docstring"""


								if type(UpperCamelCase__					) == dict:
											A__       =    Namespace(**UpperCamelCase__					)
								A__       =    glue_output_modes[hparams.task]
								A__       =    glue_tasks_num_labels[hparams.task]

								super().__init__(UpperCamelCase__				,					UpperCamelCase__				,					self.mode					)



					def     _a							(     self					:       Dict				,					**_snake_case					:       str					):






								"""simple docstring"""


								return self.model(**UpperCamelCase__					)



					def     _a							(     self					:       str				,					_snake_case					:       Tuple				,					_snake_case					:       Tuple					):






								"""simple docstring"""


								A__       =    {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}

								if self.config.model_type not in ["distilbert", "bart"]:
											A__       =    batch[2] if self.config.model_type in ['''bert''', '''xlnet''', '''albert'''] else None

								A__       =    self(**UpperCamelCase__					)
								A__       =    outputs[0]

								A__       =    self.trainer.lr_schedulers[0]['''scheduler''']
								A__       =    {'''loss''': loss, '''rate''': lr_scheduler.get_last_lr()[-1]}
								return {"loss": loss, "log": tensorboard_logs}



					def     _a							(     self					:       Optional[int]					):






								"""simple docstring"""


								A__       =    self.hparams
								A__       =    processors[args.task]()
								A__       =    processor.get_labels()

								for mode in ["train", "dev"]:
											A__       =    self._feature_file(UpperCamelCase__					)
											if os.path.exists(UpperCamelCase__					) and not args.overwrite_cache:
														logger.info('Loading features from cached file %s'				,					UpperCamelCase__					)
											else:
														logger.info('Creating features from dataset file at %s'				,					args.data_dir					)
														A__       =    (
														    processor.get_dev_examples(args.data_dir					)
														    if mode == '''dev'''
														    else processor.get_train_examples(args.data_dir					)
														)
														A__       =    convert_examples_to_features(
														    UpperCamelCase__				,					self.tokenizer				,					max_length=args.max_seq_length				,					label_list=self.labels				,					output_mode=args.glue_output_mode				,					)
														logger.info('Saving features into cached file %s'				,					UpperCamelCase__					)
														torch.save(UpperCamelCase__				,					UpperCamelCase__					)



					def     _a							(     self					:       Any				,					_snake_case					:       Optional[Any]				,					_snake_case					:       Union[str, Any]				,					_snake_case					:       List[Any] = False					):






								"""simple docstring"""


								A__       =    '''dev''' if mode == '''test''' else mode

								A__       =    self._feature_file(UpperCamelCase__					)
								logger.info('Loading features from cached file %s'				,					UpperCamelCase__					)
								A__       =    torch.load(UpperCamelCase__					)
								A__       =    torch.tensor([f.input_ids for f in features]				,					dtype=torch.long					)
								A__       =    torch.tensor([f.attention_mask for f in features]				,					dtype=torch.long					)
								A__       =    torch.tensor([f.token_type_ids for f in features]				,					dtype=torch.long					)
								if self.hparams.glue_output_mode == "classification":
											A__       =    torch.tensor([f.label for f in features]				,					dtype=torch.long					)
								elif self.hparams.glue_output_mode == "regression":
											A__       =    torch.tensor([f.label for f in features]				,					dtype=torch.float					)

								return DataLoader(
								    TensorDataset(UpperCamelCase__				,					UpperCamelCase__				,					UpperCamelCase__				,					UpperCamelCase__					)				,					batch_size=UpperCamelCase__				,					shuffle=UpperCamelCase__				,					)



					def     _a							(     self					:       Dict				,					_snake_case					:       Tuple				,					_snake_case					:       Union[str, Any]					):






								"""simple docstring"""


								A__       =    {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}

								if self.config.model_type not in ["distilbert", "bart"]:
											A__       =    batch[2] if self.config.model_type in ['''bert''', '''xlnet''', '''albert'''] else None

								A__       =    self(**UpperCamelCase__					)
								A__       =    outputs[:2]
								A__       =    logits.detach().cpu().numpy()
								A__       =    inputs['''labels'''].detach().cpu().numpy()

								return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}



					def     _a							(     self					:       Union[str, Any]				,					_snake_case					:       int					):






								"""simple docstring"""


								A__       =    torch.stack([x['val_loss'] for x in outputs]					).mean().detach().cpu().item()
								A__       =    np.concatenate([x['pred'] for x in outputs]				,					axis=0					)

								if self.hparams.glue_output_mode == "classification":
											A__       =    np.argmax(UpperCamelCase__				,					axis=1					)
								elif self.hparams.glue_output_mode == "regression":
											A__       =    np.squeeze(UpperCamelCase__					)

								A__       =    np.concatenate([x['target'] for x in outputs]				,					axis=0					)
								A__       =    [[] for _ in range(out_label_ids.shape[0]					)]
								A__       =    [[] for _ in range(out_label_ids.shape[0]					)]

								A__       =    {**{'''val_loss''': val_loss_mean}, **compute_metrics(self.hparams.task				,					UpperCamelCase__				,					UpperCamelCase__					)}

								A__       =    dict(results.items()					)
								A__       =    results
								return ret, preds_list, out_label_list



					def     _a							(     self					:       List[Any]				,					_snake_case					:       Optional[Any]					):






								"""simple docstring"""


								A__       =    self._eval_end(UpperCamelCase__					)
								A__       =    ret['''log''']
								return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}



					def     _a							(     self					:       Tuple				,					_snake_case					:       Tuple					):






								"""simple docstring"""


								A__       =    self._eval_end(UpperCamelCase__					)
								A__       =    ret['''log''']
								# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
								return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}



					@staticmethod
					def     _a							(     _snake_case					:       Any				,					_snake_case					:       List[str]					):






								"""simple docstring"""


								BaseTransformer.add_model_specific_args(UpperCamelCase__				,					UpperCamelCase__					)
								parser.add_argument(
								    '--max_seq_length'				,					default=1_28				,					type=UpperCamelCase__				,					help=(
								        'The maximum total input sequence length after tokenization. Sequences longer '
								        'than this will be truncated, sequences shorter will be padded.'
								    )				,					)

								parser.add_argument(
								    '--task'				,					default=''				,					type=UpperCamelCase__				,					required=UpperCamelCase__				,					help='The GLUE task to run'				,					)
								parser.add_argument(
								    '--gpus'				,					default=0				,					type=UpperCamelCase__				,					help='The number of GPUs allocated for this, it is by default 0 meaning none'				,					)

								parser.add_argument(
								    '--overwrite_cache'				,					action='store_true'				,					help='Overwrite the cached training and evaluation sets'					)

								return parser







def A						(       )						->      Dict:
			A__       =    argparse.ArgumentParser()
			add_generic_args(__UpperCamelCase ,	os.getcwd()					)
			A__       =    GLUETransformer.add_model_specific_args(__UpperCamelCase ,	os.getcwd()					)
			A__       =    parser.parse_args()

			# If output_dir not provided, a folder will be generated in pwd
			if args.output_dir is None:
						A__       =    os.path.join(
						    './results' ,	f'''{args.task}_{time.strftime("%Y%m%d_%H%M%S"					)}''' ,	)
						os.makedirs(args.output_dir					)

			A__       =    GLUETransformer(__UpperCamelCase					)
			A__       =    generic_train(__UpperCamelCase ,	__UpperCamelCase					)

			# Optionally, predict on dev set and write to output_dir
			if args.do_predict:
						A__       =    sorted(glob.glob(os.path.join(args.output_dir ,	'checkpoint-epoch=*.ckpt'					) ,	recursive=__UpperCamelCase					)					)
						A__       =    model.load_from_checkpoint(checkpoints[-1]					)
						return trainer.test(__UpperCamelCase					)


if __name__ == "__main__":
							main()









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__       =      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       =      {
    '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/config.json''',
    '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/config.json''',
    '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/config.json''',
    '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/config.json''',
    '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/config.json''',
    '''roberta-large-openai-detector''': '''https://huggingface.co/roberta-large-openai-detector/resolve/main/config.json''',
}


class      __lowerCAmelCase     (		UpperCAmelCase_      ):
					"""simple docstring"""






					A__				:		List[str]			       =    "roberta"



					def __init__(     self					:       List[str]				,					_snake_case					:       Union[str, Any]=5_02_65				,					_snake_case					:       List[Any]=7_68				,					_snake_case					:       List[str]=12				,					_snake_case					:       List[str]=12				,					_snake_case					:       Any=30_72				,					_snake_case					:       Union[str, Any]="gelu"				,					_snake_case					:       int=0.1				,					_snake_case					:       Union[str, Any]=0.1				,					_snake_case					:       Tuple=5_12				,					_snake_case					:       Union[str, Any]=2				,					_snake_case					:       Any=0.02				,					_snake_case					:       Any=1E-12				,					_snake_case					:       List[Any]=1				,					_snake_case					:       int=0				,					_snake_case					:       Any=2				,					_snake_case					:       Optional[Any]="absolute"				,					_snake_case					:       int=True				,					_snake_case					:       Any=None				,					**_snake_case					:       Any				,					):






								"""simple docstring"""


								super().__init__(pad_token_id=_snake_case				,					bos_token_id=_snake_case				,					eos_token_id=_snake_case				,					**_snake_case					)

								A__       =    vocab_size
								A__       =    hidden_size
								A__       =    num_hidden_layers
								A__       =    num_attention_heads
								A__       =    hidden_act
								A__       =    intermediate_size
								A__       =    hidden_dropout_prob
								A__       =    attention_probs_dropout_prob
								A__       =    max_position_embeddings
								A__       =    type_vocab_size
								A__       =    initializer_range
								A__       =    layer_norm_eps
								A__       =    position_embedding_type
								A__       =    use_cache
								A__       =    classifier_dropout







class      __lowerCAmelCase     (		UpperCAmelCase_      ):
					"""simple docstring"""






					@property
					def     _a							(     self					:       Dict					):






								"""simple docstring"""


								if self.task == "multiple-choice":
											A__       =    {0: 'batch', 1: 'choice', 2: 'sequence'}
								else:
											A__       =    {0: 'batch', 1: 'sequence'}
								return OrderedDict(
								    [
								        ('input_ids', dynamic_axis),
								        ('attention_mask', dynamic_axis),
								    ]					)




