




import argparse
import glob
import logging
import os
from argparse import Namespace
from importlib import import_module

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, TensorDataset
from utils_ner import TokenClassificationTask


__lowerCamelCase   :						Dict				=			logging.getLogger(__name__)


class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								='''token-classification'''

			def __init__( self			:   Optional[int]       ,						lowerCamelCase_			:   List[Any]  )						->   List[str]:
								if type(lowerCamelCase_  ) == dict:
													__magic_name__							:	List[str]    							=							Namespace(**lowerCamelCase_  )
								__magic_name__							:	int    							=							import_module('''tasks'''  )
								try:
													__magic_name__							:	Union[str, Any]    							=							getattr(lowerCamelCase_       ,						hparams.task_type  )
													__magic_name__							:	TokenClassificationTask    							=							token_classification_task_clazz()
								except AttributeError:
													raise ValueError(
													    F'''Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. '''
													    F'''Available tasks classes are: {TokenClassificationTask.__subclasses__()}'''  )
								__magic_name__							:	Union[str, Any]    							=							self.token_classification_task.get_labels(hparams.labels  )
								__magic_name__							:	Optional[int]    							=							CrossEntropyLoss().ignore_index
								super().__init__(lowerCamelCase_       ,						len(self.labels  )       ,						self.mode  )

			def 							UpperCAmelCase__       ( self			:   Dict       ,						**lowerCamelCase_			:   Optional[int]  )						->   Union[str, Any]:
								return self.model(**lowerCamelCase_  )

			def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   List[Any]       ,						lowerCamelCase_			:   Union[str, Any]  )						->   List[str]:
								__magic_name__							:	List[str]    							=							{'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}
								if self.config.model_type != "distilbert":
													__magic_name__							:	List[str]    							=							(
													    batch[2] if self.config.model_type in ['''bert''', '''xlnet'''] else None
													)  # XLM and RoBERTa don"t use token_type_ids

								__magic_name__							:	Any    							=							self(**lowerCamelCase_  )
								__magic_name__							:	int    							=							outputs[0]
								# tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
								return {"loss": loss}

			def 							UpperCAmelCase__       ( self			:   int  )						->   Optional[int]:
								__magic_name__							:	Dict    							=							self.hparams
								for mode in ["train", "dev", "test"]:
													__magic_name__							:	Union[str, Any]    							=							self._feature_file(lowerCamelCase_  )
													if os.path.exists(lowerCamelCase_  ) and not args.overwrite_cache:
																		logger.info('''Loading features from cached file %s'''       ,						lowerCamelCase_  )
																		__magic_name__							:	Optional[int]    							=							torch.load(lowerCamelCase_  )
													else:
																		logger.info('''Creating features from dataset file at %s'''       ,						args.data_dir  )
																		__magic_name__							:	Any    							=							self.token_classification_task.read_examples_from_file(args.data_dir       ,						lowerCamelCase_  )
																		__magic_name__							:	str    							=							self.token_classification_task.convert_examples_to_features(
																		    lowerCamelCase_       ,						self.labels       ,						args.max_seq_length       ,						self.tokenizer       ,						cls_token_at_end=bool(self.config.model_type in ['''xlnet''']  )       ,						cls_token=self.tokenizer.cls_token       ,						cls_token_segment_id=2 if self.config.model_type in ['''xlnet'''] else 0       ,						sep_token=self.tokenizer.sep_token       ,						sep_token_extra=lowerCamelCase_       ,						pad_on_left=bool(self.config.model_type in ['''xlnet''']  )       ,						pad_token=self.tokenizer.pad_token_id       ,						pad_token_segment_id=self.tokenizer.pad_token_type_id       ,						pad_token_label_id=self.pad_token_label_id       ,						)
																		logger.info('''Saving features into cached file %s'''       ,						lowerCamelCase_  )
																		torch.save(lowerCamelCase_       ,						lowerCamelCase_  )

			def 							UpperCAmelCase__       ( self			:   Any       ,						lowerCamelCase_			:   int       ,						lowerCamelCase_			:   int       ,						lowerCamelCase_			:   bool = False  )						->   DataLoader:
								__magic_name__							:	Optional[int]    							=							self._feature_file(lowerCamelCase_  )
								logger.info('''Loading features from cached file %s'''       ,						lowerCamelCase_  )
								__magic_name__							:	Optional[Any]    							=							torch.load(lowerCamelCase_  )
								__magic_name__							:	Optional[Any]    							=							torch.tensor([f.input_ids for f in features]       ,						dtype=torch.long  )
								__magic_name__							:	int    							=							torch.tensor([f.attention_mask for f in features]       ,						dtype=torch.long  )
								if features[0].token_type_ids is not None:
													__magic_name__							:	Optional[Any]    							=							torch.tensor([f.token_type_ids for f in features]       ,						dtype=torch.long  )
								else:
													__magic_name__							:	str    							=							torch.tensor([0 for f in features]       ,						dtype=torch.long  )
													# HACK(we will not use this anymore soon)
								__magic_name__							:	List[Any]    							=							torch.tensor([f.label_ids for f in features]       ,						dtype=torch.long  )
								return DataLoader(
								    TensorDataset(lowerCamelCase_       ,						lowerCamelCase_       ,						lowerCamelCase_       ,						lowerCamelCase_  )       ,						batch_size=lowerCamelCase_  )

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]       ,						lowerCamelCase_			:   Union[str, Any]       ,						lowerCamelCase_			:   Dict  )						->   int:
								"""Compute validation""" ""
								__magic_name__							:	Tuple    							=							{'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}
								if self.config.model_type != "distilbert":
													__magic_name__							:	int    							=							(
													    batch[2] if self.config.model_type in ['''bert''', '''xlnet'''] else None
													)  # XLM and RoBERTa don"t use token_type_ids
								__magic_name__							:	str    							=							self(**lowerCamelCase_  )
								__magic_name__							,    __magic_name__							:	List[Any]    							=							outputs[:2]
								__magic_name__							:	Dict    							=							logits.detach().cpu().numpy()
								__magic_name__							:	Union[str, Any]    							=							inputs['''labels'''].detach().cpu().numpy()
								return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}

			def 							UpperCAmelCase__       ( self			:   Optional[int]       ,						lowerCamelCase_			:   str  )						->   Any:
								__magic_name__							:	Optional[int]    							=							torch.stack([x['''val_loss'''] for x in outputs]  ).mean()
								__magic_name__							:	str    							=							np.concatenate([x['''pred'''] for x in outputs]       ,						axis=0  )
								__magic_name__							:	Any    							=							np.argmax(lowerCamelCase_       ,						axis=2  )
								__magic_name__							:	str    							=							np.concatenate([x['''target'''] for x in outputs]       ,						axis=0  )

								__magic_name__							:	Dict    							=							dict(enumerate(self.labels  )  )
								__magic_name__							:	Tuple    							=							[[] for _ in range(out_label_ids.shape[0]  )]
								__magic_name__							:	Any    							=							[[] for _ in range(out_label_ids.shape[0]  )]

								for i in range(out_label_ids.shape[0]  ):
													for j in range(out_label_ids.shape[1]  ):
																		if out_label_ids[i, j] != self.pad_token_label_id:
																							out_label_list[i].append(label_map[out_label_ids[i][j]]  )
																							preds_list[i].append(label_map[preds[i][j]]  )

								__magic_name__							:	List[str]    							=							{
								    '''val_loss''': val_loss_mean,
								    '''accuracy_score''': accuracy_score(lowerCamelCase_       ,						lowerCamelCase_  ),
								    '''precision''': precision_score(lowerCamelCase_       ,						lowerCamelCase_  ),
								    '''recall''': recall_score(lowerCamelCase_       ,						lowerCamelCase_  ),
								    '''f1''': fa_score(lowerCamelCase_       ,						lowerCamelCase_  ),
								}

								__magic_name__							:	List[Any]    							=							dict(results.items()  )
								__magic_name__							:	Any    							=							results
								return ret, preds_list, out_label_list

			def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   List[Any]  )						->   Dict:
								# when stable
								__magic_name__							,    __magic_name__							,    __magic_name__							:	Optional[Any]    							=							self._eval_end(lowerCamelCase_  )
								__magic_name__							:	int    							=							ret['''log''']
								return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}

			def 							UpperCAmelCase__       ( self			:   Tuple       ,						lowerCamelCase_			:   Optional[int]  )						->   Union[str, Any]:
								# updating to test_epoch_end instead of deprecated test_end
								__magic_name__							,    __magic_name__							,    __magic_name__							:	str    							=							self._eval_end(lowerCamelCase_  )

								# Converting to the dict required by pl
								# https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
								# pytorch_lightning/trainer/logging.py#L139
								__magic_name__							:	Optional[int]    							=							ret['''log''']
								# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
								return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}


			@staticmethod
			def 							UpperCAmelCase__       ( lowerCamelCase_			:   Union[str, Any]       ,						lowerCamelCase_			:   Tuple  )						->   Tuple:
								# Add NER specific options
								BaseTransformer.add_model_specific_args(lowerCamelCase_       ,						lowerCamelCase_  )
								parser.add_argument(
								    '''--task_type'''       ,						default='''NER'''       ,						type=lowerCamelCase_       ,						help='''Task type to fine tune in training (e.g. NER, POS, etc)'''  )
								parser.add_argument(
								    '''--max_seq_length'''       ,						default=128       ,						type=lowerCamelCase_       ,						help=(
								        '''The maximum total input sequence length after tokenization. Sequences longer '''
								        '''than this will be truncated, sequences shorter will be padded.'''
								    )       ,						)

								parser.add_argument(
								    '''--labels'''       ,						default=''''''       ,						type=lowerCamelCase_       ,						help='''Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.'''       ,						)
								parser.add_argument(
								    '''--gpus'''       ,						default=0       ,						type=lowerCamelCase_       ,						help='''The number of GPUs allocated for this, it is by default 0 meaning none'''       ,						)

								parser.add_argument(
								    '''--overwrite_cache'''       ,						action='''store_true'''       ,						help='''Overwrite the cached training and evaluation sets'''  )

								return parser


if __name__ == "__main__":
	__lowerCamelCase   :						Dict				=			argparse.ArgumentParser()
	add_generic_args(parser, os.getcwd())
	__lowerCamelCase   :						List[Any]				=			NERTransformer.add_model_specific_args(parser, os.getcwd())
	__lowerCamelCase   :						Any				=			parser.parse_args()
	__lowerCamelCase   :						List[Any]				=			NERTransformer(args)
	__lowerCamelCase   :						Optional[int]				=			generic_train(model, args)

	if args.do_predict:
		# See https://github.com/huggingface/transformers/issues/3159
		# pl use this default format to create a checkpoint:
		# https://github.com/PyTorchLightning/pytorch-lightning/blob/master\
		# /pytorch_lightning/callbacks/model_checkpoint.py#L322
		__lowerCamelCase   :						Union[str, Any]				=			sorted(glob.glob(os.path.join(args.output_dir, '''checkpoint-epoch=*.ckpt'''), recursive=True))
		__lowerCamelCase   :						Optional[Any]				=			model.load_from_checkpoint(checkpoints[-1])
		trainer.test(model)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase   :						Optional[int]				=			logging.get_logger(__name__)

__lowerCamelCase   :						Dict				=			{}


class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								='''llama'''
			UpperCamelCase__								=['''past_key_values''']

			def __init__( self			:   Dict       ,						lowerCamelCase_			:   List[str]=32000       ,						lowerCamelCase_			:   List[Any]=4096       ,						lowerCamelCase_			:   Any=11008       ,						lowerCamelCase_			:   Optional[Any]=32       ,						lowerCamelCase_			:   Dict=32       ,						lowerCamelCase_			:   Optional[Any]=None       ,						lowerCamelCase_			:   Dict="silu"       ,						lowerCamelCase_			:   Optional[int]=2048       ,						lowerCamelCase_			:   int=0.0_2       ,						lowerCamelCase_			:   Tuple=1E-6       ,						lowerCamelCase_			:   Tuple=True       ,						lowerCamelCase_			:   List[str]=0       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Optional[int]=2       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Dict=False       ,						lowerCamelCase_			:   int=None       ,						**lowerCamelCase_			:   Tuple       ,						)						->   Tuple:
								__magic_name__							:	Any    							=							vocab_size
								__magic_name__							:	Optional[int]    							=							max_position_embeddings
								__magic_name__							:	Union[str, Any]    							=							hidden_size
								__magic_name__							:	Dict    							=							intermediate_size
								__magic_name__							:	Union[str, Any]    							=							num_hidden_layers
								__magic_name__							:	Tuple    							=							num_attention_heads

								# for backward compatibility
								if num_key_value_heads is None:
													__magic_name__							:	Optional[int]    							=							num_attention_heads

								__magic_name__							:	Optional[int]    							=							num_key_value_heads
								__magic_name__							:	List[str]    							=							hidden_act
								__magic_name__							:	List[str]    							=							initializer_range
								__magic_name__							:	Tuple    							=							rms_norm_eps
								__magic_name__							:	Optional[int]    							=							pretraining_tp
								__magic_name__							:	Any    							=							use_cache
								__magic_name__							:	int    							=							rope_scaling
								self._rope_scaling_validation()

								super().__init__(
								    pad_token_id=lowerCamelCase_       ,						bos_token_id=lowerCamelCase_       ,						eos_token_id=lowerCamelCase_       ,						tie_word_embeddings=lowerCamelCase_       ,						**lowerCamelCase_       ,						)


			def 							UpperCAmelCase__       ( self			:   str  )						->   Tuple:
								if self.rope_scaling is None:
													return

								if not isinstance(self.rope_scaling       ,						lowerCamelCase_  ) or len(self.rope_scaling  ) != 2:
													raise ValueError(
													    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
													    F'''got {self.rope_scaling}'''  )
								__magic_name__							:	List[str]    							=							self.rope_scaling.get('''type'''       ,						lowerCamelCase_  )
								__magic_name__							:	str    							=							self.rope_scaling.get('''factor'''       ,						lowerCamelCase_  )
								if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
													raise ValueError(
													    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''  )
								if rope_scaling_factor is None or not isinstance(lowerCamelCase_       ,						lowerCamelCase_  ) or rope_scaling_factor <= 1.0:
													raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''  )



