






'''simple docstring'''





import argparse
import json
from typing import List

from ltp import LTP

from transformers.models.bert.tokenization_bert import BertTokenizer





def 					lowercase_ (    __snake_case     :							List[Any]      )     -> List[Any]:





				'''simple docstring'''


				if (
				    (cp >= 0X4_e00 and cp <= 0X9_fff)
				    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
				    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
				    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
				    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
				    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
				    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
				    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
				):  #
								return True

				return False





def 					lowercase_ (    __snake_case     :							List[Any]      )     -> Dict:





				'''simple docstring'''


				for char in word:
								snake_case__      :Tuple         =   ord(lowerCAmelCase_      )
								if not _is_chinese_char(lowerCAmelCase_      ):
												return 0
				return 1





def 					lowercase_ (    __snake_case     :							Any      )     -> Dict:





				'''simple docstring'''


				snake_case__      :Tuple         =   set()

				for token in tokens:
								snake_case__      :Tuple         =   len(lowerCAmelCase_      ) > 1 and is_chinese(lowerCAmelCase_      )
								if chinese_word:
												word_set.add(lowerCAmelCase_      )
				snake_case__      :Any         =   list(lowerCAmelCase_      )
				return word_list





def 					lowercase_ (    __snake_case     :							List[str]	,	__snake_case     :							Dict      )     -> Union[str, Any]:





				'''simple docstring'''


				if not chinese_word_set:
								return bert_tokens
				snake_case__      :str         =   max([len(lowerCAmelCase_      ) for w in chinese_word_set]      )

				snake_case__      :Tuple         =   bert_tokens
				snake_case__      :Dict         =   0, len(lowerCAmelCase_      )
				while start < end:
								snake_case__      :Any         =   True
								if is_chinese(bert_word[start]      ):
												snake_case__      :int         =   min(end - start	,	lowerCAmelCase_      )
												for i in range(lowerCAmelCase_	,	1	,	-1      ):
																snake_case__      :int         =   ''''''.join(bert_word[start : start + i]      )
																if whole_word in chinese_word_set:
																				for j in range(start + 1	,	start + i      ):
																								snake_case__      :Optional[int]         =   '''##''' + bert_word[j]
																				snake_case__      :Dict         =   start + i
																				snake_case__      :Dict         =   False
																				break
								if single_word:
												start += 1
				return bert_word





def 					lowercase_ (    __snake_case     :							Optional[Any]	,	__snake_case     :							str	,	__snake_case     :							List[str]      )     -> Tuple:





				'''simple docstring'''


				snake_case__      :List[Any]         =   []

				for i in range(0	,	len(lowerCAmelCase_      )	,	1_00      ):
								snake_case__      :Any         =   ltp_tokenizer.pipeline(lines[i : i + 1_00]	,	tasks=["cws"]      ).cws
								snake_case__      :List[Any]         =   [get_chinese_word(lowerCAmelCase_      ) for r in res]
								ltp_res.extend(lowerCAmelCase_      )
				assert len(lowerCAmelCase_      ) == len(lowerCAmelCase_      )

				snake_case__      :Dict         =   []
				for i in range(0	,	len(lowerCAmelCase_      )	,	1_00      ):
								snake_case__      :Dict         =   bert_tokenizer(lines[i : i + 1_00]	,	add_special_tokens=lowerCAmelCase_	,	truncation=lowerCAmelCase_	,	max_length=5_12      )
								bert_res.extend(res["input_ids"]      )
				assert len(lowerCAmelCase_      ) == len(lowerCAmelCase_      )

				snake_case__      :Optional[Any]         =   []
				for input_ids, chinese_word in zip(lowerCAmelCase_	,	lowerCAmelCase_      ):
								snake_case__      :Dict         =   []
								for id in input_ids:
												snake_case__      :Optional[int]         =   bert_tokenizer._convert_id_to_token(lowerCAmelCase_      )
												input_tokens.append(lowerCAmelCase_      )
								snake_case__      :Dict         =   add_sub_symbol(lowerCAmelCase_	,	lowerCAmelCase_      )
								snake_case__      :Tuple         =   []
								# We only save pos of chinese subwords start with ##, which mean is part of a whole word.
								for i, token in enumerate(lowerCAmelCase_      ):
												if token[:2] == "##":
																snake_case__      :Tuple         =   token[2:]
																# save chinese tokens' pos
																if len(lowerCAmelCase_      ) == 1 and _is_chinese_char(ord(lowerCAmelCase_      )      ):
																				ref_id.append(lowerCAmelCase_      )
								ref_ids.append(lowerCAmelCase_      )

				assert len(lowerCAmelCase_      ) == len(lowerCAmelCase_      )

				return ref_ids





def 					lowercase_ (    __snake_case     :							Union[str, Any]      )     -> Optional[int]:





				'''simple docstring'''


				with open(args.file_name	,	"r"	,	encoding="utf-8"      ) as f:
								snake_case__      :int         =   f.readlines()
				snake_case__      :int         =   [line.strip() for line in data if len(lowerCAmelCase_      ) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
				snake_case__      :Union[str, Any]         =   LTP(args.ltp      )  # faster in GPU device
				snake_case__      :Any         =   BertTokenizer.from_pretrained(args.bert      )

				snake_case__      :Tuple         =   prepare_ref(lowerCAmelCase_	,	lowerCAmelCase_	,	lowerCAmelCase_      )

				with open(args.save_path	,	"w"	,	encoding="utf-8"      ) as f:
								snake_case__      :Dict         =   [json.dumps(lowerCAmelCase_      ) + '''\n''' for ref in ref_ids]
								f.writelines(lowerCAmelCase_      )


if __name__ == "__main__":
	__UpperCAmelCase							:   Union[str, Any]							=  argparse.ArgumentParser(description="prepare_chinese_ref")
	parser.add_argument(
	    "--file_name",
	    required=False,
	    type=str,
	    default="./resources/chinese-demo.txt",
	    help="file need process, same as training data in lm",
	)
	parser.add_argument(
	    "--ltp",
	    required=False,
	    type=str,
	    default="./resources/ltp",
	    help="resources for LTP tokenizer, usually a path",
	)
	parser.add_argument(
	    "--bert",
	    required=False,
	    type=str,
	    default="./resources/robert",
	    help="resources for Bert tokenizer",
	)
	parser.add_argument(
	    "--save_path",
	    required=False,
	    type=str,
	    default="./resources/ref.txt",
	    help="path to save res",
	)

	__UpperCAmelCase							:   Union[str, Any]							=  parser.parse_args()
	main(args)







import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
	from cookiecutter.main import cookiecutter

	__UpperCAmelCase							:   Dict							=  True
except ImportError:
	__UpperCAmelCase							:   List[Any]							=  False

__UpperCAmelCase							:   Dict							=  logging.get_logger(__name__)  # pylint: disable=invalid-name





def 					lowercase_ (    __snake_case     :							Namespace      )     -> Dict:





				'''simple docstring'''


				return AddNewModelCommand(args.testing	,	args.testing_file	,	path=args.path      )


class 					_snake_case						(			_A ):






	@staticmethod
	def    lowerCAmelCase_							(		UpperCamelCase       )						->    Any:
					snake_case__      :Dict         =   parser.add_parser("add-new-model"       )
					add_new_model_parser.add_argument("--testing"					,action="store_true"					,help="If in testing mode."       )
					add_new_model_parser.add_argument("--testing_file"					,type=UpperCamelCase					,help="Configuration file on which to run."       )
					add_new_model_parser.add_argument(
					    "--path"					,type=UpperCamelCase					,help="Path to cookiecutter. Should only be used for testing purposes."       )
					add_new_model_parser.set_defaults(func=UpperCamelCase       )






	def __init__(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase=None					,*UpperCamelCase       )						->    Any:
					snake_case__      :Union[str, Any]         =   testing
					snake_case__      :Union[str, Any]         =   testing_file
					snake_case__      :List[str]         =   path






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					warnings.warn(
					    "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
					    "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
					    "checks, you should use `transformers-cli add-new-model-like` instead."       )
					if not _has_cookiecutter:
									raise ImportError(
									    "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
									    "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"       )
					# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
					snake_case__      :Tuple         =   [directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
					if len(UpperCamelCase       ) > 0:
									raise ValueError(
									    "Several directories starting with `cookiecutter-template-` in current working directory. "
									    "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
									    "change your working directory."       )

					snake_case__      :str         =   (
					    Path(UpperCamelCase       ).parent.parent.parent.parent if self._path is None else Path(self._path       ).parent.parent
					)
					snake_case__      :Tuple         =   path_to_transformer_root / "templates" / "adding_a_new_model"

					# Execute cookiecutter
					if not self._testing:
									cookiecutter(str(UpperCamelCase       )       )
					else:
									with open(self._testing_file					,"r"       ) as configuration_file:
													snake_case__      :str         =   json.load(UpperCamelCase       )

									cookiecutter(
									    str(path_to_cookiecutter if self._path is None else self._path       )					,no_input=UpperCamelCase					,extra_context=UpperCamelCase					,)

					snake_case__      :List[Any]         =   [directory for directory in os.listdir() if "cookiecutter-template-" in directory[:22]][0]

					# Retrieve configuration
					with open(directory + "/configuration.json"					,"r"       ) as configuration_file:
									snake_case__      :Dict         =   json.load(UpperCamelCase       )

					snake_case__      :Optional[Any]         =   configuration["lowercase_modelname"]
					snake_case__      :List[Any]         =   configuration["generate_tensorflow_pytorch_and_flax"]
					os.remove(f'{directory}/configuration.json'       )

					snake_case__      :Any         =   "PyTorch" in generate_tensorflow_pytorch_and_flax
					snake_case__      :Any         =   "TensorFlow" in generate_tensorflow_pytorch_and_flax
					snake_case__      :Any         =   "Flax" in generate_tensorflow_pytorch_and_flax

					snake_case__      :Dict         =   f'{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'
					os.makedirs(UpperCamelCase					,exist_ok=UpperCamelCase       )
					os.makedirs(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}'					,exist_ok=UpperCamelCase       )

					# Tests require submodules as they have parent imports
					with open(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'					,"w"       ):
									pass

					shutil.move(
					    f'{directory}/__init__.py'					,f'{model_dir}/__init__.py'					,)
					shutil.move(
					    f'{directory}/configuration_{lowercase_model_name}.py'					,f'{model_dir}/configuration_{lowercase_model_name}.py'					,)

					def remove_copy_lines(UpperCamelCase       ):
									with open(UpperCamelCase					,"r"       ) as f:
													snake_case__      :List[str]         =   f.readlines()
									with open(UpperCamelCase					,"w"       ) as f:
													for line in lines:
																	if "# Copied from transformers." not in line:
																					f.write(UpperCamelCase       )

					if output_pytorch:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_{lowercase_model_name}.py'					,f'{model_dir}/modeling_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_{lowercase_model_name}.py'       )

					if output_tensorflow:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_tf_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_tf_{lowercase_model_name}.py'					,f'{model_dir}/modeling_tf_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_tf_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_tf_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_tf_{lowercase_model_name}.py'       )

					if output_flax:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_flax_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_flax_{lowercase_model_name}.py'					,f'{model_dir}/modeling_flax_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_flax_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_flax_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_flax_{lowercase_model_name}.py'       )

					shutil.move(
					    f'{directory}/{lowercase_model_name}.md'					,f'{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'					,)

					shutil.move(
					    f'{directory}/tokenization_{lowercase_model_name}.py'					,f'{model_dir}/tokenization_{lowercase_model_name}.py'					,)

					shutil.move(
					    f'{directory}/tokenization_fast_{lowercase_model_name}.py'					,f'{model_dir}/tokenization_{lowercase_model_name}_fast.py'					,)

					from os import fdopen, remove
					from shutil import copymode, move
					from tempfile import mkstemp

					def replace(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       ):
									# Create temp file
									snake_case__    ,						snake_case__      :Optional[Any]         =   mkstemp()
									snake_case__      :Optional[Any]         =   False
									with fdopen(UpperCamelCase					,"w"       ) as new_file:
													with open(UpperCamelCase       ) as old_file:
																	for line in old_file:
																					new_file.write(UpperCamelCase       )
																					if line_to_copy_below in line:
																									snake_case__      :Optional[Any]         =   True
																									for line_to_copy in lines_to_copy:
																													new_file.write(UpperCamelCase       )

									if not line_found:
													raise ValueError(f'Line {line_to_copy_below} was not found in file.'       )

									# Copy the file permissions from the old file to the new file
									copymode(UpperCamelCase					,UpperCamelCase       )
									# Remove original file
									remove(UpperCamelCase       )
									# Move new file
									move(UpperCamelCase					,UpperCamelCase       )

					def skip_units(UpperCamelCase       ):
									return (
									    ("generating PyTorch" in line and not output_pytorch)
									    or ("generating TensorFlow" in line and not output_tensorflow)
									    or ("generating Flax" in line and not output_flax)
									)

					def replace_in_files(UpperCamelCase       ):
									with open(UpperCamelCase       ) as datafile:
													snake_case__      :int         =   []
													snake_case__      :Optional[int]         =   False
													snake_case__      :List[str]         =   False
													for line in datafile:
																	if "# To replace in: " in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   line.split("\""       )[1]
																					snake_case__      :Tuple         =   skip_units(UpperCamelCase       )
																	elif "# Below: " in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   line.split("\""       )[1]
																					snake_case__      :List[str]         =   skip_units(UpperCamelCase       )
																	elif "# End." in line and "##" not in line:
																					if not skip_file and not skip_snippet:
																									replace(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )

																					snake_case__      :Tuple         =   []
																	elif "# Replace with" in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   []
																	elif "##" not in line:
																					lines_to_copy.append(UpperCamelCase       )

									remove(UpperCamelCase       )

					replace_in_files(f'{directory}/to_replace_{lowercase_model_name}.py'       )
					os.rmdir(UpperCamelCase       )