






import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
from utils_multiple_choice import MultipleChoiceDataset, Split, processors

import transformers
from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


lowerCAmelCase			:							Tuple     =logging.getLogger(__name__)







def 						A__  (					__A   , __A	):







    '''simple docstring'''
    return (preds == labels).mean()




@dataclass
class 	__snake_case	:





      '''simple docstring'''






      _snake_case    		=       field(
          metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}		)
      _snake_case    		=       field(
          default=__lowerCamelCase ,  metadata={'help': 'Pretrained config name or path if not the same as model_name'}		)
      _snake_case    		=       field(
          default=__lowerCamelCase ,  metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}		)
      _snake_case    		=       field(
          default=__lowerCamelCase ,  metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'} ,  )





@dataclass
class 	__snake_case	:





      '''simple docstring'''






      _snake_case    		=       field(metadata={'help': 'The name of the task to train on: ' + ', '.join(processors.keys()		)}		)
      _snake_case    		=       field(metadata={'help': 'Should contain the data files for the task.'}		)
      _snake_case    		=       field(
          default=128 ,  metadata={
              'help': (
                  'The maximum total input sequence length after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )
          } ,  )
      _snake_case    		=       field(
          default=__lowerCamelCase ,  metadata={'help': 'Overwrite the cached training and evaluation sets'}		)







def 						A__  (					):







    '''simple docstring'''
    _lowerCamelCase     :		str 						=						HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)	)
    _lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase     :		Dict 						=						parser.parse_args_into_dataclasses()

    if (
        os.path.exists(training_args.output_dir	)
        and os.listdir(training_args.output_dir	)
        and training_args.do_train
        and not training_args.overwrite_output_dir
    ):
        raise ValueError(
            F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
            """ --overwrite_output_dir to overcome."""	)

    # Setup logging
    logging.basicConfig(
        format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""   , datefmt="""%m/%d/%Y %H:%M:%S"""   , level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN   , )
    logger.warning(
        """Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"""   , training_args.local_rank   , training_args.device   , training_args.n_gpu   , bool(training_args.local_rank != -1	)   , training_args.fpaa   , )
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank	):
        transformers.utils.logging.set_verbosity_info()
        transformers.utils.logging.enable_default_handler()
        transformers.utils.logging.enable_explicit_format()
    logger.info("""Training/evaluation parameters %s"""   , A__	)

    # Set seed
    set_seed(training_args.seed	)

    try:
        _lowerCamelCase     :		Union[str, Any] 						=						processors[data_args.task_name]()
        _lowerCamelCase     :		Optional[Any] 						=						processor.get_labels()
        _lowerCamelCase     :		str 						=						len(A__	)
    except KeyError:
        raise ValueError("""Task not found: %s""" % (data_args.task_name)	)

    # Load pretrained model and tokenizer
    #
    # Distributed training:
    # The .from_pretrained methods guarantee that only one local process can concurrently
    # download model & vocab.

    _lowerCamelCase     :		int 						=						AutoConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path   , num_labels=A__   , finetuning_task=data_args.task_name   , cache_dir=model_args.cache_dir   , )
    _lowerCamelCase     :		Any 						=						AutoTokenizer.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path   , cache_dir=model_args.cache_dir   , )
    _lowerCamelCase     :		Union[str, Any] 						=						AutoModelForMultipleChoice.from_pretrained(
        model_args.model_name_or_path   , from_tf=bool(""".ckpt""" in model_args.model_name_or_path	)   , config=A__   , cache_dir=model_args.cache_dir   , )

    # Get datasets
    _lowerCamelCase     :		Union[str, Any] 						=						(
        MultipleChoiceDataset(
            data_dir=data_args.data_dir   , tokenizer=A__   , task=data_args.task_name   , max_seq_length=data_args.max_seq_length   , overwrite_cache=data_args.overwrite_cache   , mode=Split.train   , )
        if training_args.do_train
        else None
    )
    _lowerCamelCase     :		Union[str, Any] 						=						(
        MultipleChoiceDataset(
            data_dir=data_args.data_dir   , tokenizer=A__   , task=data_args.task_name   , max_seq_length=data_args.max_seq_length   , overwrite_cache=data_args.overwrite_cache   , mode=Split.dev   , )
        if training_args.do_eval
        else None
    )

    def compute_metrics(__A	) -> Dict:
        _lowerCamelCase     :		Union[str, Any] 						=						np.argmax(p.predictions   , axis=1	)
        return {"acc": simple_accuracy(A__   , p.label_ids	)}

    # Data collator
    _lowerCamelCase     :		str 						=						DataCollatorWithPadding(A__   , pad_to_multiple_of=8	) if training_args.fpaa else None

    # Initialize our Trainer
    _lowerCamelCase     :		List[Any] 						=						Trainer(
        model=A__   , args=A__   , train_dataset=A__   , eval_dataset=A__   , compute_metrics=A__   , data_collator=A__   , )

    # Training
    if training_args.do_train:
        trainer.train(
            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path	) else None	)
        trainer.save_model()
        # For convenience, we also re-save the tokenizer to the same directory,
        # so that you can share your model easily on huggingface.co/models =)
        if trainer.is_world_master():
            tokenizer.save_pretrained(training_args.output_dir	)

    # Evaluation
    _lowerCamelCase     :		Dict 						=						{}
    if training_args.do_eval:
        logger.info("""*** Evaluate ***"""	)

        _lowerCamelCase     :		Any 						=						trainer.evaluate()

        _lowerCamelCase     :		str 						=						os.path.join(training_args.output_dir   , """eval_results.txt"""	)
        if trainer.is_world_master():
            with open(A__   , """w"""	) as writer:
                logger.info("""***** Eval results *****"""	)
                for key, value in result.items():
                    logger.info("""  %s = %s"""   , A__   , A__	)
                    writer.write("""%s = %s\n""" % (key, value)	)

                results.update(A__	)

    return results







def 						A__  (					__A	):







    '''simple docstring'''
    main()


if __name__ == "__main__":
    main()





import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser







def 						A__  (					__A	):







    '''simple docstring'''
    _lowerCamelCase     :		Tuple 						=						{}
    _lowerCamelCase     :		List[Any] 						=						tokenizer(example["""content"""]   , truncation=__A	)["""input_ids"""]
    _lowerCamelCase     :		Tuple 						=						len(example["""content"""]	) / len(output["""input_ids"""]	)
    return output


lowerCAmelCase			:							int     =HfArgumentParser(PretokenizationArguments)
lowerCAmelCase			:							int     =parser.parse_args()
if args.num_workers is None:
    lowerCAmelCase			:							Any     =multiprocessing.cpu_count()
lowerCAmelCase			:							Optional[Any]     =AutoTokenizer.from_pretrained(args.tokenizer_dir)

lowerCAmelCase			:							str     =time.time()
lowerCAmelCase			:							Union[str, Any]     =load_dataset(args.dataset_name, split="train")
print(F"""Dataset loaded in {time.time()-t_start:.2f}s""")

lowerCAmelCase			:							Dict     =time.time()
lowerCAmelCase			:							Dict     =ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        "repo_name",
        "path",
        "copies",
        "size",
        "content",
        "license",
        "hash",
        "line_mean",
        "line_max",
        "alpha_frac",
        "autogenerated",
    ],
)
print(F"""Dataset tokenized in {time.time()-t_start:.2f}s""")

lowerCAmelCase			:							Tuple     =time.time()
ds.push_to_hub(args.tokenized_data_repo)
print(F"""Data pushed to the hub in {time.time()-t_start:.2f}s""")




