




import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


_UpperCAmelCase	:						int			=     logging.getLogger(__name__)

_UpperCAmelCase	:						str			=     "pytorch_model.bin"




@dataclasses.dataclass
class    __lowerCAmelCase       :
       _a 					=						dataclasses.field(
           metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models.'''})
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co.'''}      , )







@dataclasses.dataclass
class    __lowerCAmelCase       :
       _a 					=						dataclasses.field(metadata={'''help''': '''A csv or a json file containing the training data.'''})
       _a 					=						dataclasses.field(metadata={'''help''': '''A csv or a json file containing the data to predict on.'''})
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''A csv or a json file containing the validation data.'''})
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''The name of the task to train on.'''}      , )
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''The list of labels for the task.'''})







@dataclasses.dataclass
class    __lowerCAmelCase       :
       _a 					=						dataclasses.field(
           metadata={'''help''': '''The output directory where the model predictions and checkpoints will be written.'''})
       _a 					=						dataclasses.field(
           default='''accuracy'''      , metadata={'''help''': '''The evaluation metric used for the task.'''})
       _a 					=						dataclasses.field(
           default='''no'''      , metadata={
               '''help''': '''The evaluation strategy to adopt during training. Possible values are: ["no", "step", "epoch]'''
           }      , )
       _a 					=						dataclasses.field(
           default=10      , metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}      , )
       _a 					=						dataclasses.field(
           default=0.0      , metadata={
               '''help''': '''How much the specified evaluation metric must improve to satisfy early stopping conditions.'''
           }      , )
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the confidence score.'''}      , )
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the validation performance.'''}      , )
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''Whether to fine-tune on labeled data after pseudo training.'''}      , )
       _a 					=						dataclasses.field(
           default=0.0      , metadata={'''help''': '''Confidence threshold for pseudo-labeled data filtering.'''}      , )
       _a 					=						dataclasses.field(
           default=100      , metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}      , )
       _a 					=						dataclasses.field(
           default=lowerCAmelCase      , metadata={'''help''': '''Random seed for initialization.'''}      , )







def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  lowercase    :str      	=						datasets.concatenate_datasets([infer_input, infer_output],			axis=1   )

  if args.do_filter_by_confidence:
    lowercase    :Optional[Any]      	=						dataset.filter(lambda lowerCamelCase   : example["probability"] > args.confidence_threshold   )

  if args.do_filter_by_val_performance:
    assert eval_result >= 0.0 and eval_result <= 1.0
    lowercase    :Optional[int]      	=						int(eval_result * len(lowerCamelCase   )   )
    print(lowerCamelCase   )
    lowercase    :int      	=						dataset.sort("probability",			reverse=lowerCamelCase   )
    lowercase    :Tuple      	=						dataset.select(range(lowerCamelCase   )   )

  lowercase    :Optional[int]      	=						dataset.remove_columns(["label", "probability"]   )
  lowercase    :Any      	=						dataset.rename_column("prediction",			"label"   )
  lowercase    :List[Any]      	=						dataset.map(lambda lowerCamelCase   : {"label": idalabel[example["label"]]}   )
  lowercase    :int      	=						dataset.shuffle(seed=args.seed   )

  lowercase    :Any      	=						os.path.join(lowerCamelCase,			F"train_pseudo.{args.data_file_extension}"   )
  if args.data_file_extension == "csv":
    dataset.to_csv(lowerCamelCase,			index=lowerCamelCase   )
  else:
    dataset.to_json(lowerCamelCase   )







def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			**lowerCamelCase   ):
  lowercase    :Tuple      	=						Accelerator()
  # Make one log on every process with the configuration for debugging.
  logging.basicConfig(
      format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",			datefmt="%m/%d/%Y %H:%M:%S",			level=logging.INFO,			)
  logger.info(accelerator.state   )

  # Setup logging, we only want one process per machine to log things on the
  # screen. accelerator.is_local_main_process is only True for one process per
  # machine.
  logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR   )

  if accelerator.is_local_main_process:
    datasets.utils.logging.set_verbosity_warning()
    transformers.utils.logging.set_verbosity_info()
  else:
    datasets.utils.logging.set_verbosity_error()
    transformers.utils.logging.set_verbosity_error()

  lowercase    :Optional[int]      	=						STModelArguments(model_name_or_path=lowerCamelCase   )
  lowercase    :Any      	=						STDataArguments(train_file=lowerCamelCase,			infer_file=lowerCamelCase   )
  lowercase    :Optional[Any]      	=						STTrainingArguments(output_dir=lowerCamelCase   )
  lowercase    :Dict      	=						argparse.Namespace()

  for arg_class in (model_args, data_args, training_args):
    for key, value in vars(lowerCamelCase   ).items():
      setattr(lowerCamelCase,			lowerCamelCase,			lowerCamelCase   )

  for key, value in kwargs.items():
    if hasattr(lowerCamelCase,			lowerCamelCase   ):
      setattr(lowerCamelCase,			lowerCamelCase,			lowerCamelCase   )

    # Sanity checks
  lowercase    :Dict      	=						{}
  lowercase    :str      	=						None

  # You need to provide the training data and the data to predict on
  assert args.train_file is not None
  assert args.infer_file is not None
  lowercase    :Optional[int]      	=						args.train_file
  lowercase    :Dict      	=						args.infer_file

  if args.evaluation_strategy != IntervalStrategy.NO.value:
    assert args.eval_file is not None
    lowercase    :Optional[Any]      	=						args.eval_file

  for key in data_files:
    lowercase    :Dict      	=						data_files[key].split("."   )[-1]
    assert extension in ["csv", "json"], F"`{key}_file` should be a csv or a json file."
    if args.data_file_extension is None:
      lowercase    :List[str]      	=						extension
    else:
      assert extension == args.data_file_extension, F"`{key}_file` should be a {args.data_file_extension} file`."

  assert (
      args.eval_metric in datasets.list_metrics()
  ), F"{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}."

  # If passed along, set the training seed now.
  if args.seed is not None:
    set_seed(args.seed   )

  logger.info("Creating the initial data directory for self-training..."   )
  lowercase    :Tuple      	=						F"{args.output_dir}/self-train_iter-{{}}".format
  lowercase    :int      	=						data_dir_format(0   )

  if accelerator.is_main_process:
    if args.output_dir is not None:
      os.makedirs(args.output_dir,			exist_ok=lowerCamelCase   )
      os.makedirs(lowerCamelCase,			exist_ok=lowerCamelCase   )
  accelerator.wait_for_everyone()

  lowercase    :Dict      	=						None
  lowercase    :Tuple      	=						None
  lowercase    :Tuple      	=						0
  lowercase    :Dict      	=						False
  # Show the progress bar
  lowercase    :List[str]      	=						tqdm(range(args.max_selftrain_iterations   ),			disable=not accelerator.is_local_main_process   )

  # Self-train
  for iteration in range(0,			int(args.max_selftrain_iterations   )   ):
    lowercase    :Dict      	=						data_dir_format(lowerCamelCase   )
    assert os.path.exists(lowerCamelCase   )

    # Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
    # iteration > 0
    lowercase    :str      	=						os.path.join(lowerCamelCase,			"stage-1"   )
    lowercase    :int      	=						{
        "accelerator": accelerator,
        "model_name_or_path": args.model_name_or_path,
        "cache_dir": args.cache_dir,
        "do_train": True,
        "train_file": data_files["train"] if iteration == 0 else data_files["train_pseudo"],
        "do_eval": True if args.eval_file is not None else False,
        "eval_file": data_files["eval"],
        "do_predict": True,
        "infer_file": data_files["infer"],
        "task_name": args.task_name,
        "label_list": args.label_list,
        "output_dir": current_output_dir,
        "eval_metric": args.eval_metric,
        "evaluation_strategy": args.evaluation_strategy,
        "early_stopping_patience": args.early_stopping_patience,
        "early_stopping_threshold": args.early_stopping_threshold,
        "seed": args.seed,
    }
    # Add additional training arguments
    for key, value in kwargs.items():
      if key not in arguments_dict and not hasattr(lowerCamelCase,			lowerCamelCase   ):
        arguments_dict.update({key: value}   )

    lowercase    :Union[str, Any]      	=						os.path.join(lowerCamelCase,			"best-checkpoint",			lowerCamelCase   )
    if os.path.exists(lowerCamelCase   ):
      logger.info(
          "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1.",			lowerCamelCase,			lowerCamelCase,			)
    else:
      logger.info("***** Running self-training: iteration: %d, stage: 1 *****",			lowerCamelCase   )
      finetune(**lowerCamelCase   )
      accelerator.wait_for_everyone()
      assert os.path.exists(lowerCamelCase   )
      logger.info("Self-training job completed: iteration: %d, stage: 1.",			lowerCamelCase   )

    if iteration > 0 and args.finetune_on_labeled_data:
      # Stage 2 (optional): fine-tuning on the original labeled data
      lowercase    :int      	=						os.path.join(lowerCamelCase,			"best-checkpoint"   )
      lowercase    :str      	=						os.path.join(lowerCamelCase,			"stage-2"   )
      # Update arguments_dict
      lowercase    :Dict      	=						model_path
      lowercase    :Dict      	=						data_files["train"]
      lowercase    :Optional[Any]      	=						current_output_dir

      lowercase    :List[str]      	=						os.path.join(lowerCamelCase,			"best-checkpoint",			lowerCamelCase   )
      if os.path.exists(lowerCamelCase   ):
        logger.info(
            "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2.",			lowerCamelCase,			lowerCamelCase,			)
      else:
        logger.info("***** Running self-training: iteration: %d, stage: 2 *****",			lowerCamelCase   )
        finetune(**lowerCamelCase   )
        accelerator.wait_for_everyone()
        assert os.path.exists(lowerCamelCase   )
        logger.info("Self-training job completed: iteration: %d, stage: 2.",			lowerCamelCase   )

    lowercase    :Optional[int]      	=						iteration
    lowercase    :str      	=						data_dir_format(iteration + 1   )

    lowercase    :str      	=						AutoConfig.from_pretrained(os.path.join(lowerCamelCase,			"best-checkpoint"   )   )
    lowercase    :Any      	=						config.idalabel
    lowercase    :int      	=						os.path.join(lowerCamelCase,			"eval_results_best-checkpoint.json"   )
    lowercase    :List[str]      	=						os.path.join(lowerCamelCase,			"test_results_best-checkpoint.json"   )
    assert os.path.exists(lowerCamelCase   )

    with open(lowerCamelCase,			"r"   ) as f:
      lowercase    :Union[str, Any]      	=						float(json.load(lowerCamelCase   )[args.eval_metric]   )
    lowercase    :List[Any]      	=						os.path.join(lowerCamelCase,			"infer_output_best-checkpoint.csv"   )
    assert os.path.exists(lowerCamelCase   )
    # Loading the dataset from local csv or json files.
    lowercase    :int      	=						load_dataset(args.data_file_extension,			data_files={"data": data_files["infer"]}   )["data"]
    lowercase    :Tuple      	=						load_dataset("csv",			data_files={"data": infer_output_file}   )["data"]

    if accelerator.is_main_process:
      os.makedirs(lowerCamelCase,			exist_ok=lowerCamelCase   )
      shutil.copy(lowerCamelCase,			os.path.join(lowerCamelCase,			F"eval_results_iter-{iteration}.json"   )   )
      if os.path.exists(lowerCamelCase   ):
        shutil.copy(lowerCamelCase,			os.path.join(lowerCamelCase,			F"test_results_iter-{iteration}.json"   )   )
      create_pseudo_labeled_data(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   )
    accelerator.wait_for_everyone()

    lowercase    :List[Any]      	=						os.path.join(lowerCamelCase,			F"train_pseudo.{args.data_file_extension}"   )

    if args.evaluation_strategy != IntervalStrategy.NO.value:
      lowercase    :Tuple      	=						eval_result

      if best_iteration is None:
        lowercase    :Any      	=						new_iteration
        lowercase    :Union[str, Any]      	=						new_eval_result
      else:
        if new_eval_result - best_eval_result > args.early_stopping_threshold:
          lowercase    :str      	=						new_iteration
          lowercase    :Union[str, Any]      	=						new_eval_result
          lowercase    :Union[str, Any]      	=						0
        else:
          if new_eval_result == best_eval_result:
            lowercase    :Optional[Any]      	=						new_iteration
            lowercase    :Optional[int]      	=						new_eval_result
          early_stopping_patience_counter += 1

        if early_stopping_patience_counter >= args.early_stopping_patience:
          lowercase    :Union[str, Any]      	=						True

    progress_bar.update(1   )

    if should_training_stop:
      break

  if best_iteration is not None:
    # Save the best iteration
    logger.info("Best iteration: %d",			lowerCamelCase   )
    logger.info("Best evaluation result: %s = %f",			args.eval_metric,			lowerCamelCase   )
    accelerator.wait_for_everyone()
    if accelerator.is_main_process:
      shutil.copy(
          os.path.join(lowerCamelCase,			F"eval_results_iter-{iteration}.json"   ),			os.path.join(lowerCamelCase,			"eval_results_best-iteration.json"   ),			)
  else:
    # Assume that the last iteration is the best
    logger.info("Best iteration: %d",			args.max_selftrain_iterations - 1   )
    logger.info("Best evaluation result: %s = %f",			args.eval_metric,			lowerCamelCase   )
    accelerator.wait_for_everyone()
    if accelerator.is_main_process:
      shutil.copy(
          os.path.join(lowerCamelCase,			F"eval_results_iter-{args.max_selftrain_iterations - 1}.json"   ),			os.path.join(lowerCamelCase,			"eval_results_best-iteration.json"   ),			)









_UpperCAmelCase	:						Union[str, Any]			=     "0.21.0"

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
     from .utils import rich



