


"""simple docstring"""

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
       import torch


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


UpperCAmelCase__  							=logging.get_logger(__name__)
UpperCAmelCase__  							={"vocab_file": "spiece.model"}

UpperCAmelCase__  							={
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

UpperCAmelCase__  							={
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}





class    lowerCamelCase__						(		_a    ):
      a :     Union[str, Any]											=		VOCAB_FILES_NAMES
      a :     List[Any]											=		PRETRAINED_VOCAB_FILES_MAP
      a :     Dict											=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a :     Optional[int]											=		["""input_ids""", """attention_mask"""]
      def __init__(		self	:	Any      ,       A_	:	Optional[int]      ,       A_	:	Union[str, Any]=False      ,       A_	:	Dict=False      ,       A_	:	str=False      ,       A_	:	Any=None      ,       A_	:	Dict=None      ,       A_	:	Optional[int]=None      ,       A_	:	List[Any]=None      ,       A_	:	Optional[Dict[str, Any]] = None      ,       **A_	:	Union[str, Any]      ,       ):






       '''simple docstring'''


       __lowercase									=	{} if sp_model_kwargs is None else sp_model_kwargs

       __lowercase									=	kwargs.get("""name_or_path"""							)
       if name_or_path is None:
        logger.warning(
            """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
            """ you are testing the model, this can safely be ignored"""							)
        __lowercase									=	"""None"""

       # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
       __lowercase									=	"""<|endoftext|>""" if eos_token is None else eos_token
       __lowercase									=	"""<unk>""" if unk_token is None else unk_token
       if "gpt-sw3-7b" in name_or_path:
        __lowercase									=	unk_token if pad_token is None else pad_token
        __lowercase									=	eos_token if bos_token is None else bos_token
       else:
        __lowercase									=	"""<pad>""" if pad_token is None else pad_token
        __lowercase									=	"""<s>""" if bos_token is None else bos_token

       super().__init__(
           do_lower_case=A_      ,       remove_space=A_      ,       keep_accents=A_      ,       bos_token=A_      ,       eos_token=A_      ,       unk_token=A_      ,       pad_token=A_      ,       sp_model_kwargs=self.sp_model_kwargs      ,       **A_      ,       )

       __lowercase									=	do_lower_case
       __lowercase									=	remove_space
       __lowercase									=	keep_accents
       __lowercase									=	vocab_file

       __lowercase									=	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
       self.sp_model.Load(A_							)

       # Used for whitespace normalization in input texts
       # fmt : off
       __lowercase									=	{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
       # fmt : on

       # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
       __lowercase									=	re.compile(
           F'''[{''.join(map(A_      ,       list(range(0      ,       9							)							) + list(range(1_1      ,       3_2							)							) + list(range(1_2_7      ,       1_6_0							)							) + [1_6_0, 1_7_3, 8_2_0_3]							)							)}]'''							)
      def __getstate__(		self	:	Optional[Any]							):






       '''simple docstring'''


       __lowercase									=	self.__dict__.copy()
       __lowercase									=	None
       return state
      def __setstate__(		self	:	Union[str, Any]      ,       A_	:	Dict							):






       '''simple docstring'''


       __lowercase									=	d

       # for backward compatibility
       if not hasattr(self      ,       """sp_model_kwargs"""							):
        __lowercase									=	{}

       __lowercase									=	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
       self.sp_model.Load(self.vocab_file							)
      @property
      # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       return len(self.sp_model							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Any      ,       A_	:	str							):






       '''simple docstring'''


       __lowercase									=	self.non_printing_characters_re.sub(""""""      ,       A_							)

       # Normalize whitespaces
       __lowercase									=	"""""".join([char if char not in self.whitespaces else """ """ for char in text]							)

       # NFC Unicode normalization
       __lowercase									=	unicodedata.normalize("""NFC"""      ,       A_							)
       return text
      def SCREAMING_SNAKE_CASE_	(		self	:	Dict      ,       A_	:	str      ,       **A_	:	Optional[int]							):






       '''simple docstring'''


       __lowercase									=	self.preprocess_text(A_							)
       return self.sp_model.encode(A_      ,       out_type=A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple      ,       A_	:	str							):






       '''simple docstring'''


       return self.sp_model.PieceToId(A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Dict      ,       A_	:	int							):






       '''simple docstring'''


       return self.sp_model.IdToPiece(A_							)
      @staticmethod
      def SCREAMING_SNAKE_CASE_	(		A_	:	str							):






       '''simple docstring'''


       return out_string
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       A_	:	List[str]							):






       '''simple docstring'''


       __lowercase									=	[]
       __lowercase									=	""""""
       __lowercase									=	False
       for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
         # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
         if not prev_is_special:
          out_string += " "

         out_string += self.sp_model.decode(A_							) + token
         __lowercase									=	True
         __lowercase									=	[]
        else:
         current_sub_tokens.append(A_							)
         __lowercase									=	False
       out_string += self.sp_model.decode(A_							)

       return out_string
      def SCREAMING_SNAKE_CASE_	(		self	:	int							):






       '''simple docstring'''


       __lowercase									=	{self.convert_ids_to_tokens(A_							): i for i in range(self.vocab_size							)}
       vocab.update(self.added_tokens_encoder							)
       return vocab
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]      ,       A_	:	str      ,       A_	:	Optional[str] = None							):






       '''simple docstring'''


       if not os.path.isdir(A_							):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
        return
       __lowercase									=	os.path.join(
           A_      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

       if os.path.abspath(self.vocab_file							) != os.path.abspath(A_							) and os.path.isfile(self.vocab_file							):
        copyfile(self.vocab_file      ,       A_							)
       elif not os.path.isfile(self.vocab_file							):
        with open(A_      ,       """wb"""							) as fi:
         __lowercase									=	self.sp_model.serialized_model_proto()
         fi.write(A_							)

       return (out_vocab_file,)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]      ,       A_	:	Union[str, List[str]]      ,       A_	:	Union[str, bool] = False							):






       '''simple docstring'''


       if isinstance(A_      ,       A_							):
        __lowercase									=	self.preprocess_text(A_							)
        __lowercase									=	self.sp_model.encode(A_							)
       else:
        __lowercase									=	[self.preprocess_text(A_							) for t in text]
        __lowercase									=	self.sp_model.encode(A_							)

       if return_tensors is True or return_tensors == "pt":
        __lowercase									=	torch.tensor(A_							)

       return token_ids
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]      ,       A_	:	Union[int, List[int]]							):






       '''simple docstring'''


       return self.sp_model.decode(A_							)






      def SCREAMING_SNAKE_CASE_	(		self	:	int      ,       A_	:	"Conversation"							):






       '''simple docstring'''


       __lowercase									=	[F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
       __lowercase									=	(
           F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(A_							) + F'''{self.bos_token}Bot:'''
       )
       return self.encode(text=A_							)




"""simple docstring"""

import argparse
from copy import deepcopy

import numpy as np
from datasets import ClassLabel, DatasetDict, load_dataset
from evaluate import load

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    set_seed,
)


def lowerCAmelCase_      (				):






 """simple docstring"""







 __lowercase									=	argparse.ArgumentParser()
 parser.add_argument("""--model_ckpt"""				,   type=UpperCamelCase__				,   default="""microsoft/unixcoder-base-nine""" )
 parser.add_argument("""--num_epochs"""				,   type=UpperCamelCase__				,   default=5 )
 parser.add_argument("""--batch_size"""				,   type=UpperCamelCase__				,   default=6 )
 parser.add_argument("""--gradient_accumulation_steps"""				,   type=UpperCamelCase__				,   default=1 )
 parser.add_argument("""--freeze"""				,   type=UpperCamelCase__				,   default=UpperCamelCase__ )
 parser.add_argument("""--learning_rate"""				,   type=UpperCamelCase__				,   default=5E-4 )
 parser.add_argument("""--seed"""				,   type=UpperCamelCase__				,   default=0 )
 parser.add_argument("""--lr_scheduler_type"""				,   type=UpperCamelCase__				,   default="""cosine""" )
 parser.add_argument("""--num_warmup_steps"""				,   type=UpperCamelCase__				,   default=10 )
 parser.add_argument("""--weight_decay"""				,   type=UpperCamelCase__				,   default=0.01 )
 parser.add_argument("""--output_dir"""				,   type=UpperCamelCase__				,   default="""./results""" )
 return parser.parse_args()


UpperCAmelCase__  							=load("accuracy")


def lowerCAmelCase_      (				UpperCamelCase__	: List[Any] ):






 """simple docstring"""







 __lowercase							,		__lowercase									=	eval_pred
 __lowercase									=	np.argmax(UpperCamelCase__				,   axis=1 )
 return metric.compute(predictions=UpperCamelCase__				,   references=UpperCamelCase__ )





class    lowerCamelCase__						(		_a    ):
      def __init__(		self	:	Any      ,       A_	:	int							):






       '''simple docstring'''


       super().__init__()
       __lowercase									=	trainer






      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       A_	:	int      ,       A_	:	str      ,       A_	:	Any      ,       **A_	:	Optional[int]							):






       '''simple docstring'''


       if control.should_evaluate:
        __lowercase									=	deepcopy(A_							)
        self._trainer.evaluate(eval_dataset=self._trainer.train_dataset      ,       metric_key_prefix="""train"""							)
        return control_copy






def lowerCAmelCase_      (				):






 """simple docstring"""







 __lowercase									=	get_args()
 set_seed(args.seed )

 __lowercase									=	load_dataset("""codeparrot/codecomplex"""				,   split="""train""" )
 __lowercase									=	dataset.train_test_split(test_size=0.2 )
 __lowercase									=	train_test["""test"""].train_test_split(test_size=0.5 )
 __lowercase									=	DatasetDict(
     {
         """train""": train_test["""train"""],
         """test""": test_validation["""train"""],
         """valid""": test_validation["""test"""],
     } )

 print("""Loading tokenizer and model""" )
 __lowercase									=	AutoTokenizer.from_pretrained(args.model_ckpt )
 __lowercase									=	tokenizer.eos_token
 __lowercase									=	AutoModelForSequenceClassification.from_pretrained(args.model_ckpt				,   num_labels=7 )
 __lowercase									=	model.config.eos_token_id

 if args.freeze:
  for param in model.roberta.parameters():
   __lowercase									=	False

 __lowercase									=	ClassLabel(num_classes=7				,   names=list(set(train_test_validation["""train"""]["""complexity"""] ) ) )

 def tokenize(UpperCamelCase__	: Optional[Any] ):
  __lowercase									=	tokenizer(example["""src"""]				,   truncation=UpperCamelCase__				,   max_length=1024 )
  __lowercase									=	labels.straint(example["""complexity"""] )
  return {
      "input_ids": inputs["input_ids"],
      "attention_mask": inputs["attention_mask"],
      "label": label,
  }

 __lowercase									=	train_test_validation.map(
     UpperCamelCase__				,   batched=UpperCamelCase__				,   remove_columns=train_test_validation["""train"""].column_names				,   )
 __lowercase									=	DataCollatorWithPadding(tokenizer=UpperCamelCase__ )

 __lowercase									=	TrainingArguments(
     output_dir=args.output_dir				,   learning_rate=args.learning_rate				,   lr_scheduler_type=args.lr_scheduler_type				,   evaluation_strategy="""epoch"""				,   save_strategy="""epoch"""				,   logging_strategy="""epoch"""				,   per_device_train_batch_size=args.batch_size				,   per_device_eval_batch_size=args.batch_size				,   num_train_epochs=args.num_epochs				,   gradient_accumulation_steps=args.gradient_accumulation_steps				,   weight_decay=0.01				,   metric_for_best_model="""accuracy"""				,   run_name="""complexity-java"""				,   report_to="""wandb"""				,   )

 __lowercase									=	Trainer(
     model=UpperCamelCase__				,   args=UpperCamelCase__				,   train_dataset=tokenized_datasets["""train"""]				,   eval_dataset=tokenized_datasets["""valid"""]				,   tokenizer=UpperCamelCase__				,   data_collator=UpperCamelCase__				,   compute_metrics=UpperCamelCase__				,   )

 print("""Training...""" )
 trainer.add_callback(CustomCallback(UpperCamelCase__ ) )
 trainer.train()


if __name__ == "__main__":
       main()
