






from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml




class 			UpperCAmelCase_      (   yaml.SafeLoader       ):




							'''simple docstring'''



							def    _UpperCAmelCase     (					self       :	List[str]       ,     a       :	Union[str, Any]			)			->					Optional[Any]:
										SCREAMING_SNAKE_CASE           =					[self.constructed_objects[key_node] for key_node, _ in node.value]
										SCREAMING_SNAKE_CASE           =					[tuple(_lowercase			) if isinstance(_lowercase       ,     _lowercase			) else key for key in keys]
										SCREAMING_SNAKE_CASE           =					Counter(_lowercase			)
										SCREAMING_SNAKE_CASE           =					[key for key in counter if counter[key] > 1]
										if duplicate_keys:
													raise TypeError(f"""Got duplicate yaml keys: {duplicate_keys}"""			)







							def    _UpperCAmelCase     (					self       :	Optional[int]       ,     a       :	Tuple       ,     a       :	Dict=False			)			->					str:
										SCREAMING_SNAKE_CASE           =					super().construct_mapping(_lowercase       ,     deep=_lowercase			)
										self._check_no_duplicates_on_constructed_node(_lowercase			)
										return mapping





def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			SCREAMING_SNAKE_CASE           =					list(readme_content.splitlines()   )
			if full_content and full_content[0] == "---" and "---" in full_content[1:]:
						SCREAMING_SNAKE_CASE           =					full_content[1:].index("""---"""   ) + 1
						SCREAMING_SNAKE_CASE           =					"""\n""".join(full_content[1:sep_idx]   )
						return yamlblock, "\n".join(full_content[sep_idx + 1 :]   )

			return None, "\n".join(_lowerCamelCase   )




class 			UpperCAmelCase_      (   __snake_case       ):




							'''simple docstring'''



							a__						  =     {'''train_eval_index'''}  # train-eval-index in the YAML metadata





							@classmethod
							def    _UpperCAmelCase     (					cls       :	str       ,     a       :	List[str]			)			->					Union[str, Any]:

										with open(_lowercase       ,     encoding="""utf-8"""			) as readme_file:
													SCREAMING_SNAKE_CASE           =					_split_yaml_from_readme(readme_file.read()			)
										if yaml_string is not None:
													return cls.from_yaml_string(_lowercase			)
										else:
													return cls()





							def    _UpperCAmelCase     (					self       :	List[Any]       ,     a       :	Any			)			->					Union[str, Any]:

										if path.exists():
													with open(_lowercase       ,     encoding="""utf-8"""			) as readme_file:
																SCREAMING_SNAKE_CASE           =					readme_file.read()
										else:
													SCREAMING_SNAKE_CASE           =					None
										SCREAMING_SNAKE_CASE           =					self._to_readme(_lowercase			)
										with open(_lowercase       ,     """w"""       ,     encoding="""utf-8"""			) as readme_file:
													readme_file.write(_lowercase			)





							def    _UpperCAmelCase     (					self       :	Dict       ,     a       :	List[str] = None			)			->					Optional[int]:

										if readme_content is not None:
													SCREAMING_SNAKE_CASE           =					_split_yaml_from_readme(_lowercase			)
													SCREAMING_SNAKE_CASE           =					"""---\n""" + self.to_yaml_string() + """---\n""" + content
										else:
													SCREAMING_SNAKE_CASE           =					"""---\n""" + self.to_yaml_string() + """---\n"""
										return full_content





							@classmethod
							def    _UpperCAmelCase     (					cls       :	Optional[int]       ,     a       :	Dict			)			->					Any:
										SCREAMING_SNAKE_CASE           =					yaml.load(_lowercase       ,     Loader=_NoDuplicateSafeLoader			) or {}

										# Convert the YAML keys to DatasetMetadata fields
										SCREAMING_SNAKE_CASE           =					{
										    (key.replace("""-"""       ,     """_"""			) if key.replace("""-"""       ,     """_"""			) in cls._FIELDS_WITH_DASHES else key): value
										    for key, value in metadata_dict.items()
										}
										return cls(**_lowercase			)







							def    _UpperCAmelCase     (					self       :	Any			)			->					Optional[Any]:
										return yaml.safe_dump(
										    {
										        (key.replace("""_"""       ,     """-"""			) if key in self._FIELDS_WITH_DASHES else key): value
										        for key, value in self.items()
										    }       ,     sort_keys=_lowercase       ,     allow_unicode=_lowercase       ,     encoding="""utf-8"""       ,     ).decode("""utf-8"""			)





__A						:							Tuple							=  {
    """image-classification""": [],
    """translation""": [],
    """image-segmentation""": [],
    """fill-mask""": [],
    """automatic-speech-recognition""": [],
    """token-classification""": [],
    """sentence-similarity""": [],
    """audio-classification""": [],
    """question-answering""": [],
    """summarization""": [],
    """zero-shot-classification""": [],
    """table-to-text""": [],
    """feature-extraction""": [],
    """other""": [],
    """multiple-choice""": [],
    """text-classification""": [],
    """text-to-image""": [],
    """text2text-generation""": [],
    """zero-shot-image-classification""": [],
    """tabular-classification""": [],
    """tabular-regression""": [],
    """image-to-image""": [],
    """tabular-to-text""": [],
    """unconditional-image-generation""": [],
    """text-retrieval""": [],
    """text-to-speech""": [],
    """object-detection""": [],
    """audio-to-audio""": [],
    """text-generation""": [],
    """conversational""": [],
    """table-question-answering""": [],
    """visual-question-answering""": [],
    """image-to-text""": [],
    """reinforcement-learning""": [],
    """voice-activity-detection""": [],
    """time-series-forecasting""": [],
    """document-question-answering""": [],
}


if __name__ == "__main__":
		from argparse import ArgumentParser

		__A						:							Dict							=  ArgumentParser(usage="""Validate the yaml metadata block of a README.md file.""")
		ap.add_argument("""readme_filepath""")
		__A						:							Tuple							=  ap.parse_args()

		__A						:							Tuple							=  Path(args.readme_filepath)
		__A						:							Tuple							=  DatasetMetadata.from_readme(readme_filepath)
		print(dataset_metadata)
		dataset_metadata.to_readme(readme_filepath)












import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__A						:							Union[str, Any]							=  logging.get_logger(__name__)

__A						:							str							=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__A						:							List[Any]							=  {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

__A						:							Tuple							=  {
    """Salesforce/codegen-350M-mono""": 2_0_4_8,
}




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     VOCAB_FILES_NAMES
							a__						  =     PRETRAINED_VOCAB_FILES_MAP
							a__						  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a__						  =     ['''input_ids''', '''attention_mask''']
							a__						  =     CodeGenTokenizer





							def __init__(					self       :	Tuple       ,     a       :	Dict=None       ,     a       :	Tuple=None       ,     a       :	Tuple=None       ,     a       :	Optional[Any]="<|endoftext|>"       ,     a       :	Any="<|endoftext|>"       ,     a       :	Dict="<|endoftext|>"       ,     a       :	Optional[int]=False       ,     **a       :	Tuple       ,     )			->					str:
										super().__init__(
										    a       ,     a       ,     tokenizer_file=a       ,     unk_token=a       ,     bos_token=a       ,     eos_token=a       ,     add_prefix_space=a       ,     **a       ,     )

										if kwargs.pop("""add_bos_token"""       ,     a			):
													SCREAMING_SNAKE_CASE           =					kwargs.pop("""name_or_path"""       ,     """"""			)
													raise ValueError(
													    """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
													    """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
													    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
													    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
													    """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
													    """ so that the fast tokenizer works correctly."""			)

										SCREAMING_SNAKE_CASE           =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
										if pre_tok_state.get("""add_prefix_space"""       ,     a			) != add_prefix_space:
													SCREAMING_SNAKE_CASE           =					getattr(a       ,     pre_tok_state.pop("""type"""			)			)
													SCREAMING_SNAKE_CASE           =					add_prefix_space
													SCREAMING_SNAKE_CASE           =					pre_tok_class(**a			)

										SCREAMING_SNAKE_CASE           =					add_prefix_space





							def    _UpperCAmelCase     (					self       :	Optional[Any]       ,     *a       :	Optional[Any]       ,     **a       :	List[Any]			)			->					BatchEncoding:
										SCREAMING_SNAKE_CASE           =					kwargs.get("""is_split_into_words"""       ,     a			)
										assert self.add_prefix_space or not is_split_into_words, (
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*a       ,     **a			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     *a       :	Tuple       ,     **a       :	List[Any]			)			->					BatchEncoding:
										SCREAMING_SNAKE_CASE           =					kwargs.get("""is_split_into_words"""       ,     a			)

										assert self.add_prefix_space or not is_split_into_words, (
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*a       ,     **a			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	str       ,     a       :	Optional[str] = None			)			->					Tuple[str]:
										SCREAMING_SNAKE_CASE           =					self._tokenizer.model.save(a       ,     name=a			)
										return tuple(a			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]       ,     a       :	bool = False       ,     a       :	bool = None       ,     a       :	Optional[List[str]] = None       ,     **a       :	List[Any]       ,     )			->					str:
										SCREAMING_SNAKE_CASE           =					super().decode(
										    token_ids=a       ,     skip_special_tokens=a       ,     clean_up_tokenization_spaces=a       ,     **a       ,     )

										if truncate_before_pattern is not None and len(a			) > 0:
													SCREAMING_SNAKE_CASE           =					self.truncate(a       ,     a			)

										return decoded_text







							def    _UpperCAmelCase     (					self       :	Any       ,     a       :	Optional[Any]       ,     a       :	List[str]			)			->					Any:
										def find_re(a       :	Tuple       ,     a       :	Any       ,     a       :	List[str]			):
													SCREAMING_SNAKE_CASE           =					pattern.search(a       ,     a			)
													return m.start() if m else -1

										SCREAMING_SNAKE_CASE           =					[re.compile(a       ,     re.MULTILINE			) for pattern in truncate_before_pattern]

										SCREAMING_SNAKE_CASE           =					list(re.finditer("""^print"""       ,     a       ,     re.MULTILINE			)			)

										if len(a			) > 1:
													SCREAMING_SNAKE_CASE           =					completion[: prints[1].start()]

										SCREAMING_SNAKE_CASE           =					list(re.finditer("""^def"""       ,     a       ,     re.MULTILINE			)			)

										if len(a			) > 1:
													SCREAMING_SNAKE_CASE           =					completion[: defs[1].start()]

										SCREAMING_SNAKE_CASE           =					0

										SCREAMING_SNAKE_CASE           =					[
										    pos for pos in [find_re(a       ,     a       ,     a			) for terminal in terminals] if pos != -1
										]

										if len(a			) > 0:
													return completion[: min(a			)]
										else:
													return completion






