




import json
import logging
import math
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from datasets import Dataset, load_dataset

import transformers
from transformers import (
    CONFIG_MAPPING,
    MODEL_FOR_MASKED_LM_MAPPING,
    AutoConfig,
    AutoModelForMaskedLM,
    AutoTokenizer,
    DataCollatorForWholeWordMask,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process


UpperCAmelCase_						=logging.getLogger(__name__)
UpperCAmelCase_						=list(MODEL_FOR_MASKED_LM_MAPPING.keys())
UpperCAmelCase_						=tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)


@dataclass
class 							__UpperCamelCase			:





				'''simple docstring'''



				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={
				        """help""": (
				            """The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."""
				        )
				    }     ,					)
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """If training from scratch, pass a model type from the list: """ + """, """.join(__UpperCAmelCase  )}     ,					)
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={
				        """help""": (
				            """Override some existing default config settings when a model is trained from scratch. Example: """
				            """n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"""
				        )
				    }     ,					)
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}  )
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}  )
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}     ,					)
				__a							: bool										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."""}     ,					)
				__a							: str										=field(
				    default="""main"""     ,					metadata={"""help""": """The specific model version to use (can be a branch name, tag name or commit id)."""}     ,					)
				__a							: bool										=field(
				    default=__UpperCAmelCase     ,					metadata={
				        """help""": (
				            """Will use the token generated when running `huggingface-cli login` (necessary to use this script """
				            """with private models)."""
				        )
				    }     ,					)



				def       __snake_case		( self						):
											if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
																		raise ValueError(
																		    '''--config_overrides can\'t be used in combination with --config_name or --model_name_or_path'''						)




@dataclass
class 							__UpperCamelCase			:





				'''simple docstring'''



				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}  )
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}  )
				__a							: Optional[str]										=field(default=__UpperCAmelCase     ,					metadata={"""help""": """The input training data file (a text file)."""}  )
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """An optional input evaluation data file to evaluate the perplexity on (a text file)."""}     ,					)
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """An optional input train ref data file for whole word masking in Chinese."""}     ,					)
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """An optional input validation ref data file for whole word masking in Chinese."""}     ,					)
				__a							: bool										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Overwrite the cached training and evaluation sets"""}  )
				__a							: Optional[int]										=field(
				    default=5     ,					metadata={
				        """help""": """The percentage of the train set used as validation set in case there's no validation split"""
				    }     ,					)
				__a							: Optional[int]										=field(
				    default=__UpperCAmelCase     ,					metadata={
				        """help""": (
				            """The maximum total input sequence length after tokenization. Sequences longer """
				            """than this will be truncated. Default to the max input length of the model."""
				        )
				    }     ,					)
				__a							: Optional[int]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """The number of processes to use for the preprocessing."""}     ,					)
				__a							: float										=field(
				    default=0.15     ,					metadata={"""help""": """Ratio of tokens to mask for masked language modeling loss"""}  )
				__a							: bool										=field(
				    default=__UpperCAmelCase     ,					metadata={
				        """help""": (
				            """Whether to pad all samples to `max_seq_length`. """
				            """If False, will pad the samples dynamically when batching to the maximum length in the batch."""
				        )
				    }     ,					)



				def       __snake_case		( self						):
											if self.train_file is not None:
																		lowerCAmelCase   =		self.train_file.split('''.'''						)[-1]
																		assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
											if self.validation_file is not None:
																		lowerCAmelCase   =		self.validation_file.split('''.'''						)[-1]
																		assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."




def       UpperCAmelCase       (     _snake_case						, _snake_case							):
							with open(_snake_case						, '''r'''						, encoding='''utf-8'''							) as f:
														lowerCAmelCase   =		[json.loads(_snake_case							) for line in f.read().splitlines() if (len(_snake_case							) > 0 and not line.isspace())]
							assert len(_snake_case							) == len(_snake_case							)

							lowerCAmelCase   =		{c: dataset[c] for c in dataset.column_names}
							lowerCAmelCase   =		refs
							return Dataset.from_dict(_snake_case							)



def       UpperCAmelCase       (     ):
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							lowerCAmelCase   =		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)
							if len(sys.argv							) == 2 and sys.argv[1].endswith('''.json'''							):
														# If we pass only one argument to the script and it's the path to a json file,
														# let's parse it to get our arguments.
														lowerCAmelCase   ,     lowerCAmelCase   ,     lowerCAmelCase   =		parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]							)							)
							else:
														lowerCAmelCase   ,     lowerCAmelCase   ,     lowerCAmelCase   =		parser.parse_args_into_dataclasses()

							# Detecting last checkpoint.
							lowerCAmelCase   =		None
							if os.path.isdir(training_args.output_dir							) and training_args.do_train and not training_args.overwrite_output_dir:
														lowerCAmelCase   =		get_last_checkpoint(training_args.output_dir							)
														if last_checkpoint is None and len(os.listdir(training_args.output_dir							)							) > 0:
																					raise ValueError(
																					    F"""Output directory ({training_args.output_dir}) already exists and is not empty. """
																					    '''Use --overwrite_output_dir to overcome.'''							)
														elif last_checkpoint is not None:
																					logger.info(
																					    F"""Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change """
																					    '''the `--output_dir` or add `--overwrite_output_dir` to train from scratch.'''							)

    # Setup logging
							logging.basicConfig(
							    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''						, datefmt='''%m/%d/%Y %H:%M:%S'''						, handlers=[logging.StreamHandler(sys.stdout							)]						, )
							logger.setLevel(logging.INFO if is_main_process(training_args.local_rank							) else logging.WARN							)

							# Log on each process the small summary:
							logger.warning(
							    F"""Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"""
							    + F"""distributed training: {bool(training_args.local_rank != -1							)}, 16-bits training: {training_args.fpaa}"""							)
							# Set the verbosity to info of the Transformers logger (on main process only):
							if is_main_process(training_args.local_rank							):
														transformers.utils.logging.set_verbosity_info()
														transformers.utils.logging.enable_default_handler()
														transformers.utils.logging.enable_explicit_format()
							logger.info('''Training/evaluation parameters %s'''						, _snake_case							)

							# Set seed before initializing model.
							set_seed(training_args.seed							)

							# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
							# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
							# (the dataset will be downloaded automatically from the datasets Hub).
							#
							# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
							# 'text' is found. You can easily tweak this behavior (see below).
							#
							# In distributed training, the load_dataset function guarantee that only one local process can concurrently
							# download the dataset.
							if data_args.dataset_name is not None:
														# Downloading and loading a dataset from the hub.
														lowerCAmelCase   =		load_dataset(data_args.dataset_name						, data_args.dataset_config_name							)
														if "validation" not in datasets.keys():
																					lowerCAmelCase   =		load_dataset(
																					    data_args.dataset_name						, data_args.dataset_config_name						, split=F"""train[:{data_args.validation_split_percentage}%]"""						, )
																					lowerCAmelCase   =		load_dataset(
																					    data_args.dataset_name						, data_args.dataset_config_name						, split=F"""train[{data_args.validation_split_percentage}%:]"""						, )
							else:
														lowerCAmelCase   =		{}
														if data_args.train_file is not None:
																					lowerCAmelCase   =		data_args.train_file
														if data_args.validation_file is not None:
																					lowerCAmelCase   =		data_args.validation_file
														lowerCAmelCase   =		data_args.train_file.split('''.'''							)[-1]
														if extension == "txt":
																					lowerCAmelCase   =		'''text'''
														lowerCAmelCase   =		load_dataset(_snake_case						, data_files=_snake_case							)
							# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
							# https://huggingface.co/docs/datasets/loading_datasets.html.

							# Load pretrained model and tokenizer
							#
							# Distributed training:
							# The .from_pretrained methods guarantee that only one local process can concurrently
							# download model & vocab.
							lowerCAmelCase   =		{
							    '''cache_dir''': model_args.cache_dir,
							    '''revision''': model_args.model_revision,
							    '''use_auth_token''': True if model_args.use_auth_token else None,
							}
							if model_args.config_name:
														lowerCAmelCase   =		AutoConfig.from_pretrained(model_args.config_name						, **_snake_case							)
							elif model_args.model_name_or_path:
														lowerCAmelCase   =		AutoConfig.from_pretrained(model_args.model_name_or_path						, **_snake_case							)
							else:
														lowerCAmelCase   =		CONFIG_MAPPING[model_args.model_type]()
														logger.warning('''You are instantiating a new config instance from scratch.'''							)
														if model_args.config_overrides is not None:
																					logger.info(F"""Overriding config: {model_args.config_overrides}"""							)
																					config.update_from_string(model_args.config_overrides							)
																					logger.info(F"""New config: {config}"""							)

							lowerCAmelCase   =		{
							    '''cache_dir''': model_args.cache_dir,
							    '''use_fast''': model_args.use_fast_tokenizer,
							    '''revision''': model_args.model_revision,
							    '''use_auth_token''': True if model_args.use_auth_token else None,
							}
							if model_args.tokenizer_name:
														lowerCAmelCase   =		AutoTokenizer.from_pretrained(model_args.tokenizer_name						, **_snake_case							)
							elif model_args.model_name_or_path:
														lowerCAmelCase   =		AutoTokenizer.from_pretrained(model_args.model_name_or_path						, **_snake_case							)
							else:
														raise ValueError(
														    '''You are instantiating a new tokenizer from scratch. This is not supported by this script.'''
														    '''You can do it from another script, save it, and load it from here, using --tokenizer_name.'''							)

							if model_args.model_name_or_path:
														lowerCAmelCase   =		AutoModelForMaskedLM.from_pretrained(
														    model_args.model_name_or_path						, from_tf=bool('''.ckpt''' in model_args.model_name_or_path							)						, config=_snake_case						, cache_dir=model_args.cache_dir						, revision=model_args.model_revision						, use_auth_token=True if model_args.use_auth_token else None						, )
							else:
														logger.info('''Training new model from scratch'''							)
														lowerCAmelCase   =		AutoModelForMaskedLM.from_config(_snake_case							)

							model.resize_token_embeddings(len(_snake_case							)							)

							# Preprocessing the datasets.
							# First we tokenize all the texts.
							if training_args.do_train:
														lowerCAmelCase   =		datasets['''train'''].column_names
							else:
														lowerCAmelCase   =		datasets['''validation'''].column_names
							lowerCAmelCase   =		'''text''' if '''text''' in column_names else column_names[0]

							lowerCAmelCase   =		'''max_length''' if data_args.pad_to_max_length else False

							def tokenize_function(_snake_case							):
														# Remove empty lines
														lowerCAmelCase   =		[line for line in examples['''text'''] if len(_snake_case							) > 0 and not line.isspace()]
														return tokenizer(examples['''text''']						, padding=_snake_case						, truncation=_snake_case						, max_length=data_args.max_seq_length							)

							lowerCAmelCase   =		datasets.map(
							    _snake_case						, batched=_snake_case						, num_proc=data_args.preprocessing_num_workers						, remove_columns=[text_column_name]						, load_from_cache_file=not data_args.overwrite_cache						, )

							# Add the chinese references if provided
							if data_args.train_ref_file is not None:
														lowerCAmelCase   =		add_chinese_references(tokenized_datasets['''train''']						, data_args.train_ref_file							)
							if data_args.validation_ref_file is not None:
														lowerCAmelCase   =		add_chinese_references(
														    tokenized_datasets['''validation''']						, data_args.validation_ref_file							)
							# If we have ref files, need to avoid it removed by trainer
							lowerCAmelCase   =		data_args.train_ref_file or data_args.validation_ref_file
							if has_ref:
														lowerCAmelCase   =		False

							# Data collator
							# This one will take care of randomly masking the tokens.
							lowerCAmelCase   =		DataCollatorForWholeWordMask(tokenizer=_snake_case						, mlm_probability=data_args.mlm_probability							)

							# Initialize our Trainer
							lowerCAmelCase   =		Trainer(
							    model=_snake_case						, args=_snake_case						, train_dataset=tokenized_datasets['''train'''] if training_args.do_train else None						, eval_dataset=tokenized_datasets['''validation'''] if training_args.do_eval else None						, tokenizer=_snake_case						, data_collator=_snake_case						, )

							# Training
							if training_args.do_train:
														if last_checkpoint is not None:
																					lowerCAmelCase   =		last_checkpoint
														elif model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path							):
																					lowerCAmelCase   =		model_args.model_name_or_path
														else:
																					lowerCAmelCase   =		None
														lowerCAmelCase   =		trainer.train(resume_from_checkpoint=_snake_case							)
														trainer.save_model()  # Saves the tokenizer too for easy upload

														lowerCAmelCase   =		os.path.join(training_args.output_dir						, '''train_results.txt'''							)
														if trainer.is_world_process_zero():
																					with open(_snake_case						, '''w'''							) as writer:
																												logger.info('''***** Train results *****'''							)
																												for key, value in sorted(train_result.metrics.items()							):
																																			logger.info(F"""  {key} = {value}"""							)
																																			writer.write(F"""{key} = {value}\n"""							)

            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
																					trainer.state.save_to_json(os.path.join(training_args.output_dir						, '''trainer_state.json'''							)							)

    # Evaluation
							lowerCAmelCase   =		{}
							if training_args.do_eval:
														logger.info('''*** Evaluate ***'''							)

														lowerCAmelCase   =		trainer.evaluate()

														lowerCAmelCase   =		math.exp(eval_output['''eval_loss''']							)
														lowerCAmelCase   =		perplexity

														lowerCAmelCase   =		os.path.join(training_args.output_dir						, '''eval_results_mlm_wwm.txt'''							)
														if trainer.is_world_process_zero():
																					with open(_snake_case						, '''w'''							) as writer:
																												logger.info('''***** Eval results *****'''							)
																												for key, value in sorted(results.items()							):
																																			logger.info(F"""  {key} = {value}"""							)
																																			writer.write(F"""{key} = {value}\n"""							)

							return results



def       UpperCAmelCase       (     _snake_case							):
							# For xla_spawn (TPUs)
							main()


if __name__ == "__main__":
		main()









from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml


class 							__UpperCamelCase			( yaml.SafeLoader  ):





				'''simple docstring'''



				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		[self.constructed_objects[key_node] for key_node, _ in node.value]
											lowerCAmelCase   =		[tuple(UpperCAmelCase_						) if isinstance(UpperCAmelCase_       , UpperCAmelCase_						) else key for key in keys]
											lowerCAmelCase   =		Counter(UpperCAmelCase_						)
											lowerCAmelCase   =		[key for key in counter if counter[key] > 1]
											if duplicate_keys:
																		raise TypeError(F"""Got duplicate yaml keys: {duplicate_keys}"""						)






				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_=False						):
											lowerCAmelCase   =		super().construct_mapping(UpperCAmelCase_       , deep=UpperCAmelCase_						)
											self._check_no_duplicates_on_constructed_node(UpperCAmelCase_						)
											return mapping




def       UpperCAmelCase       (     _snake_case							):
							lowerCAmelCase   =		list(readme_content.splitlines()							)
							if full_content and full_content[0] == "---" and "---" in full_content[1:]:
														lowerCAmelCase   =		full_content[1:].index('''---'''							) + 1
														lowerCAmelCase   =		'''\n'''.join(full_content[1:sep_idx]							)
														return yamlblock, "\n".join(full_content[sep_idx + 1 :]							)

							return None, "\n".join(_snake_case							)


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Any										={"""train_eval_index"""}  # train-eval-index in the YAML metadata



				@classmethod
				def       __snake_case		( cls       , UpperCAmelCase_						):
											with open(UpperCAmelCase_       , encoding='''utf-8'''						) as readme_file:
																		lowerCAmelCase   ,     lowerCAmelCase   =		_split_yaml_from_readme(readme_file.read()						)
											if yaml_string is not None:
																		return cls.from_yaml_string(UpperCAmelCase_						)
											else:
																		return cls()



				def       __snake_case		( self       , UpperCAmelCase_						):
											if path.exists():
																		with open(UpperCAmelCase_       , encoding='''utf-8'''						) as readme_file:
																									lowerCAmelCase   =		readme_file.read()
											else:
																		lowerCAmelCase   =		None
											lowerCAmelCase   =		self._to_readme(UpperCAmelCase_						)
											with open(UpperCAmelCase_       , '''w'''       , encoding='''utf-8'''						) as readme_file:
																		readme_file.write(UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_ = None						):
											if readme_content is not None:
																		lowerCAmelCase   ,     lowerCAmelCase   =		_split_yaml_from_readme(UpperCAmelCase_						)
																		lowerCAmelCase   =		'''---\n''' + self.to_yaml_string() + '''---\n''' + content
											else:
																		lowerCAmelCase   =		'''---\n''' + self.to_yaml_string() + '''---\n'''
											return full_content



				@classmethod
				def       __snake_case		( cls       , UpperCAmelCase_						):
											lowerCAmelCase   =		yaml.load(UpperCAmelCase_       , Loader=_NoDuplicateSafeLoader						) or {}

											# Convert the YAML keys to DatasetMetadata fields
											lowerCAmelCase   =		{
											    (key.replace('''-'''       , '''_'''						) if key.replace('''-'''       , '''_'''						) in cls._FIELDS_WITH_DASHES else key): value
											    for key, value in metadata_dict.items()
											}
											return cls(**UpperCAmelCase_						)






				def       __snake_case		( self						):
											return yaml.safe_dump(
											    {
											        (key.replace('''_'''       , '''-'''						) if key in self._FIELDS_WITH_DASHES else key): value
											        for key, value in self.items()
											    }       , sort_keys=UpperCAmelCase_       , allow_unicode=UpperCAmelCase_       , encoding='''utf-8'''       , ).decode('''utf-8'''						)




UpperCAmelCase_						={
    """image-classification""": [],
    """translation""": [],
    """image-segmentation""": [],
    """fill-mask""": [],
    """automatic-speech-recognition""": [],
    """token-classification""": [],
    """sentence-similarity""": [],
    """audio-classification""": [],
    """question-answering""": [],
    """summarization""": [],
    """zero-shot-classification""": [],
    """table-to-text""": [],
    """feature-extraction""": [],
    """other""": [],
    """multiple-choice""": [],
    """text-classification""": [],
    """text-to-image""": [],
    """text2text-generation""": [],
    """zero-shot-image-classification""": [],
    """tabular-classification""": [],
    """tabular-regression""": [],
    """image-to-image""": [],
    """tabular-to-text""": [],
    """unconditional-image-generation""": [],
    """text-retrieval""": [],
    """text-to-speech""": [],
    """object-detection""": [],
    """audio-to-audio""": [],
    """text-generation""": [],
    """conversational""": [],
    """table-question-answering""": [],
    """visual-question-answering""": [],
    """image-to-text""": [],
    """reinforcement-learning""": [],
    """voice-activity-detection""": [],
    """time-series-forecasting""": [],
    """document-question-answering""": [],
}


if __name__ == "__main__":
		from argparse import ArgumentParser

		UpperCAmelCase_						=ArgumentParser(usage="""Validate the yaml metadata block of a README.md file.""")
		ap.add_argument("""readme_filepath""")
		UpperCAmelCase_						=ap.parse_args()

		UpperCAmelCase_						=Path(args.readme_filepath)
		UpperCAmelCase_						=DatasetMetadata.from_readme(readme_filepath)
		print(dataset_metadata)
		dataset_metadata.to_readme(readme_filepath)



