




"""simple docstring"""





import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
A   =       logging.getLogger(__name__)

def 					_UpperCamelCase    (			)			->			Optional[Any]:


			"""simple docstring"""
			__UpperCAmelCase	: Optional[Any]		=					argparse.ArgumentParser(
			    description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."							)
			parser.add_argument("--file_path"				,       type=UpperCamelCase				,       default="data/dump.txt"				,       help="The path to the data."							)
			parser.add_argument("--tokenizer_type"				,       type=UpperCamelCase				,       default="bert"				,       choices=["bert", "roberta", "gpt2"]							)
			parser.add_argument("--tokenizer_name"				,       type=UpperCamelCase				,       default="bert-base-uncased"				,       help="The tokenizer to use."							)
			parser.add_argument("--dump_file"				,       type=UpperCamelCase				,       default="data/dump"				,       help="The dump file prefix."							)
			__UpperCAmelCase	: Optional[Any]		=					parser.parse_args()

			logger.info(f"Loading Tokenizer ({args.tokenizer_name})"							)
			if args.tokenizer_type == "bert":
						__UpperCAmelCase	: Optional[int]		=					BertTokenizer.from_pretrained(args.tokenizer_name							)
						__UpperCAmelCase	: Optional[int]		=					tokenizer.special_tokens_map["cls_token"]  # `[CLS]`
						__UpperCAmelCase	: Optional[Any]		=					tokenizer.special_tokens_map["sep_token"]  # `[SEP]`
			elif args.tokenizer_type == "roberta":
						__UpperCAmelCase	: str		=					RobertaTokenizer.from_pretrained(args.tokenizer_name							)
						__UpperCAmelCase	: str		=					tokenizer.special_tokens_map["cls_token"]  # `<s>`
						__UpperCAmelCase	: int		=					tokenizer.special_tokens_map["sep_token"]  # `</s>`
			elif args.tokenizer_type == "gpt2":
						__UpperCAmelCase	: str		=					GPTaTokenizer.from_pretrained(args.tokenizer_name							)
						__UpperCAmelCase	: Tuple		=					tokenizer.special_tokens_map["bos_token"]  # `<|endoftext|>`
						__UpperCAmelCase	: Optional[Any]		=					tokenizer.special_tokens_map["eos_token"]  # `<|endoftext|>`

			logger.info(f"Loading text from {args.file_path}"							)
			with open(args.file_path				,       "r"				,       encoding="utf8"							) as fp:
						__UpperCAmelCase	: Tuple		=					fp.readlines()

			logger.info("Start encoding"							)
			logger.info(f"{len(UpperCamelCase							)} examples to process."							)

			__UpperCAmelCase	: Union[str, Any]		=					[]
			__UpperCAmelCase	: Any		=					0
			__UpperCAmelCase	: str		=					1_0000
			__UpperCAmelCase	: Dict		=					time.time()
			for text in data:
						__UpperCAmelCase	: str		=					f"{bos} {text.strip()} {sep}"
						__UpperCAmelCase	: int		=					tokenizer.encode(UpperCamelCase				,       add_special_tokens=UpperCamelCase							)
						rslt.append(UpperCamelCase							)

						iter += 1
						if iter % interval == 0:
									__UpperCAmelCase	: List[Any]		=					time.time()
									logger.info(f"{iter} examples processed. - {(end-start):.2f}s/{interval}expl"							)
									__UpperCAmelCase	: str		=					time.time()
			logger.info("Finished binarization"							)
			logger.info(f"{len(UpperCamelCase							)} examples processed."							)

			__UpperCAmelCase	: List[str]		=					f"{args.dump_file}.{args.tokenizer_name}.pickle"
			__UpperCAmelCase	: Any		=					tokenizer.vocab_size
			if vocab_size < (1 << 16):
						__UpperCAmelCase	: List[Any]		=					[np.uintaa(UpperCamelCase							) for d in rslt]
			else:
						__UpperCAmelCase	: int		=					[np.intaa(UpperCamelCase							) for d in rslt]
			random.shuffle(rslt_							)
			logger.info(f"Dump to {dp_file}"							)
			with open(UpperCamelCase				,       "wb"							) as handle:
						pickle.dump(rslt_				,       UpperCamelCase				,       protocol=pickle.HIGHEST_PROTOCOL							)


if __name__ == "__main__":
					main()








"""simple docstring"""





import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
A   =       """src/transformers"""
A   =       """docs/source/en"""
A   =       """."""

def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase				,       UpperCamelCase							)			->			int:


			"""simple docstring"""
			with open(UpperCamelCase				,       "r"				,       encoding="utf-8"				,       newline="\n"							) as f:
						__UpperCAmelCase	: str		=					f.readlines()
			# Find the start prompt.
			__UpperCAmelCase	: Optional[int]		=					0
			while not lines[start_index].startswith(UpperCamelCase							):
						start_index += 1
			start_index += 1

			__UpperCAmelCase	: Union[str, Any]		=					start_index
			while not lines[end_index].startswith(UpperCamelCase							):
						end_index += 1
			end_index -= 1

			while len(lines[start_index]							) <= 1:
						start_index += 1
			while len(lines[end_index]							) <= 1:
						end_index -= 1
			end_index += 1
			return "".join(lines[start_index:end_index]							), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
A   =       """Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
A   =       re.compile(r"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
A   =       re.compile(r"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
A   =       re.compile(r"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# This is to make sure the transformers module imported is the one in the repo.
A   =       direct_transformers_import(TRANSFORMERS_PATH)

def 					_UpperCamelCase    (			UpperCamelCase							)			->			str:


			"""simple docstring"""
			__UpperCAmelCase	: Tuple		=					re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"				,       UpperCamelCase							)
			return [m.group(0							) for m in matches]

def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase							)			->			Dict:


			"""simple docstring"""
			__UpperCAmelCase	: Optional[int]		=					2 if text == "✅" or text == "❌" else len(UpperCamelCase							)
			__UpperCAmelCase	: Optional[Any]		=					(width - text_length) // 2
			__UpperCAmelCase	: Tuple		=					width - text_length - left_indent
			return " " * left_indent + text + " " * right_indent

def 					_UpperCamelCase    (			)			->			Union[str, Any]:


			"""simple docstring"""
			__UpperCAmelCase	: str		=					transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
			__UpperCAmelCase	: Dict		=					{
			    name: config_maping_names[code]
			    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
			    if code in config_maping_names
			}
			__UpperCAmelCase	: Tuple		=					{name: config.replace("Config"				,       ""							) for name, config in model_name_to_config.items()}

			# Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
			__UpperCAmelCase	: int		=					collections.defaultdict(UpperCamelCase							)
			__UpperCAmelCase	: Optional[Any]		=					collections.defaultdict(UpperCamelCase							)
			__UpperCAmelCase	: Optional[int]		=					collections.defaultdict(UpperCamelCase							)
			__UpperCAmelCase	: Any		=					collections.defaultdict(UpperCamelCase							)
			__UpperCAmelCase	: str		=					collections.defaultdict(UpperCamelCase							)

			# Let's lookup through all transformers object (once).
			for attr_name in dir(UpperCamelCase							):
						__UpperCAmelCase	: Union[str, Any]		=					None
						if attr_name.endswith("Tokenizer"							):
									__UpperCAmelCase	: int		=					slow_tokenizers
									__UpperCAmelCase	: Optional[Any]		=					attr_name[:-9]
						elif attr_name.endswith("TokenizerFast"							):
									__UpperCAmelCase	: Any		=					fast_tokenizers
									__UpperCAmelCase	: Union[str, Any]		=					attr_name[:-13]
						elif _re_tf_models.match(UpperCamelCase							) is not None:
									__UpperCAmelCase	: Dict		=					tf_models
									__UpperCAmelCase	: List[str]		=					_re_tf_models.match(UpperCamelCase							).groups()[0]
						elif _re_flax_models.match(UpperCamelCase							) is not None:
									__UpperCAmelCase	: List[str]		=					flax_models
									__UpperCAmelCase	: List[str]		=					_re_flax_models.match(UpperCamelCase							).groups()[0]
						elif _re_pt_models.match(UpperCamelCase							) is not None:
									__UpperCAmelCase	: Dict		=					pt_models
									__UpperCAmelCase	: List[str]		=					_re_pt_models.match(UpperCamelCase							).groups()[0]

						if lookup_dict is not None:
									while len(UpperCamelCase							) > 0:
												if attr_name in model_name_to_prefix.values():
															__UpperCAmelCase	: List[Any]		=					True
															break
												# Try again after removing the last word in the name
												__UpperCAmelCase	: List[Any]		=					"".join(camel_case_split(UpperCamelCase							)[:-1]							)

    # Let's build that table!
			__UpperCAmelCase	: List[str]		=					list(model_name_to_config.keys()							)
			model_names.sort(key=str.lower							)
			__UpperCAmelCase	: Optional[Any]		=					["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
			# We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
			__UpperCAmelCase	: Optional[int]		=					[len(UpperCamelCase							) + 2 for c in columns]
			__UpperCAmelCase	: Tuple		=					max([len(UpperCamelCase							) for name in model_names]							) + 2

			# Build the table per se
			__UpperCAmelCase	: List[Any]		=					"|" + "|".join([_center_text(UpperCamelCase				,       UpperCamelCase							) for c, w in zip(UpperCamelCase				,       UpperCamelCase							)]							) + "|\n"
			# Use ":-----:" format to center-aligned table cell texts
			table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]							) + "|\n"

			__UpperCAmelCase	: Optional[Any]		=					{True: "✅", False: "❌"}
			for name in model_names:
						__UpperCAmelCase	: List[Any]		=					model_name_to_prefix[name]
						__UpperCAmelCase	: int		=					[
						    name,
						    check[slow_tokenizers[prefix]],
						    check[fast_tokenizers[prefix]],
						    check[pt_models[prefix]],
						    check[tf_models[prefix]],
						    check[flax_models[prefix]],
						]
						table += "|" + "|".join([_center_text(UpperCamelCase				,       UpperCamelCase							) for l, w in zip(UpperCamelCase				,       UpperCamelCase							)]							) + "|\n"
			return table

def 					_UpperCamelCase    (			UpperCamelCase=False							)			->			str:


			"""simple docstring"""
			__UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase	: Union[str, Any]		=					_find_text_in_file(
			    filename=os.path.join(UpperCamelCase				,       "index.md"							)				,       start_prompt="<!--This table is updated automatically from the auto modules"				,       end_prompt="<!-- End table-->"				,       )
			__UpperCAmelCase	: Optional[Any]		=					get_model_table_from_auto_modules()

			if current_table != new_table:
						if overwrite:
									with open(os.path.join(UpperCamelCase				,       "index.md"							)				,       "w"				,       encoding="utf-8"				,       newline="\n"							) as f:
												f.writelines(lines[:start_index] + [new_table] + lines[end_index:]							)
						else:
									raise ValueError(
									    "The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."							)


if __name__ == "__main__":
					A   =       argparse.ArgumentParser()
					parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
					A   =       parser.parse_args()

					check_model_table(args.fix_and_overwrite)


