






'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					_snake_case						(			__A       ,    unittest.TestCase ):
	_A          =   BertTokenizer
	_A          =   BertTokenizerFast
	_A          =   True
	_A          =   True
	_A          =   filter_non_english






	def    lowerCAmelCase_							(		self       )						->    int:
					super().setUp()

					snake_case__      :int         =   [
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    '[PAD]',
					    '[MASK]',
					    'want',
					    '##want',
					    '##ed',
					    'wa',
					    'un',
					    'runn',
					    '##ing',
					    ',',
					    'low',
					    'lowest',
					]
					snake_case__      :Optional[int]         =   os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["vocab_file"]       )
					with open(self.vocab_file					,"w"					,encoding="utf-8"       ) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )






	def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    str:
					snake_case__      :int         =   'UNwant\u00E9d,running'
					snake_case__      :Optional[int]         =   'unwanted, running'
					return input_text, output_text






	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :Optional[Any]         =   self.tokenizer_class(self.vocab_file       )

					snake_case__      :int         =   tokenizer.tokenize("UNwant\u00E9d,running"       )
					self.assertListEqual(UpperCamelCase					,["un", "##want", "##ed", ",", "runn", "##ing"]       )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase       )					,[9, 6, 7, 12, 10, 11]       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:





					if not self.test_rust_tokenizer:
									return

					snake_case__      :List[str]         =   self.get_tokenizer()
					snake_case__      :Optional[Any]         =   self.get_rust_tokenizer()

					snake_case__      :Optional[Any]         =   'UNwant\u00E9d,running'

					snake_case__      :str         =   tokenizer.tokenize(UpperCamelCase       )
					snake_case__      :Dict         =   rust_tokenizer.tokenize(UpperCamelCase       )
					self.assertListEqual(UpperCamelCase					,UpperCamelCase       )

					snake_case__      :int         =   tokenizer.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )
					snake_case__      :Dict         =   rust_tokenizer.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )
					self.assertListEqual(UpperCamelCase					,UpperCamelCase       )

					snake_case__      :Tuple         =   self.get_rust_tokenizer()
					snake_case__      :Optional[int]         =   tokenizer.encode(UpperCamelCase       )
					snake_case__      :Any         =   rust_tokenizer.encode(UpperCamelCase       )
					self.assertListEqual(UpperCamelCase					,UpperCamelCase       )

					# With lower casing
					snake_case__      :Union[str, Any]         =   self.get_tokenizer(do_lower_case=UpperCamelCase       )
					snake_case__      :Tuple         =   self.get_rust_tokenizer(do_lower_case=UpperCamelCase       )

					snake_case__      :List[Any]         =   'UNwant\u00E9d,running'

					snake_case__      :Optional[int]         =   tokenizer.tokenize(UpperCamelCase       )
					snake_case__      :Union[str, Any]         =   rust_tokenizer.tokenize(UpperCamelCase       )
					self.assertListEqual(UpperCamelCase					,UpperCamelCase       )

					snake_case__      :Optional[int]         =   tokenizer.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )
					snake_case__      :List[Any]         =   rust_tokenizer.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )
					self.assertListEqual(UpperCamelCase					,UpperCamelCase       )

					snake_case__      :str         =   self.get_rust_tokenizer()
					snake_case__      :Tuple         =   tokenizer.encode(UpperCamelCase       )
					snake_case__      :Union[str, Any]         =   rust_tokenizer.encode(UpperCamelCase       )
					self.assertListEqual(UpperCamelCase					,UpperCamelCase       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__      :Tuple         =   BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )					,["ah", "\u535A", "\u63A8", "zz"]       )






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :Any         =   BasicTokenizer(do_lower_case=UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )					,["hello", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,["hello"]       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					snake_case__      :Optional[Any]         =   BasicTokenizer(do_lower_case=UpperCamelCase					,strip_accents=UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,["hällo", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,["h\u00E9llo"]       )






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :Optional[Any]         =   BasicTokenizer(do_lower_case=UpperCamelCase					,strip_accents=UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,["hallo", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,["hello"]       )






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :Optional[int]         =   BasicTokenizer(do_lower_case=UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,["hallo", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,["hello"]       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:
					snake_case__      :List[Any]         =   BasicTokenizer(do_lower_case=UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )					,["HeLLo", "!", "how", "Are", "yoU", "?"]       )






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :Optional[Any]         =   BasicTokenizer(do_lower_case=UpperCamelCase					,strip_accents=UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,["HäLLo", "!", "how", "Are", "yoU", "?"]       )






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :Dict         =   BasicTokenizer(do_lower_case=UpperCamelCase					,strip_accents=UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,["HaLLo", "!", "how", "Are", "yoU", "?"]       )






	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :Optional[Any]         =   BasicTokenizer(do_lower_case=UpperCamelCase					,never_split=["[UNK]"]       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )					,["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :Any         =   BasicTokenizer()
					snake_case__      :int         =   'a\n\'ll !!to?\'d of, can\'t.'
					snake_case__      :Any         =   ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
					self.assertListEqual(tokenizer.tokenize(UpperCamelCase       )					,UpperCamelCase       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:
					snake_case__      :Optional[Any]         =   ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

					snake_case__      :Optional[Any]         =   {}
					for i, token in enumerate(UpperCamelCase       ):
									snake_case__      :Dict         =   i
					snake_case__      :Any         =   WordpieceTokenizer(vocab=UpperCamelCase					,unk_token="[UNK]"       )

					self.assertListEqual(tokenizer.tokenize(""       )					,[]       )

					self.assertListEqual(tokenizer.tokenize("unwanted running"       )					,["un", "##want", "##ed", "runn", "##ing"]       )

					self.assertListEqual(tokenizer.tokenize("unwantedX running"       )					,["[UNK]", "runn", "##ing"]       )






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					self.assertTrue(_is_whitespace(" "       )       )
					self.assertTrue(_is_whitespace("\t"       )       )
					self.assertTrue(_is_whitespace("\r"       )       )
					self.assertTrue(_is_whitespace("\n"       )       )
					self.assertTrue(_is_whitespace("\u00A0"       )       )

					self.assertFalse(_is_whitespace("A"       )       )
					self.assertFalse(_is_whitespace("-"       )       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:
					self.assertTrue(_is_control("\u0005"       )       )

					self.assertFalse(_is_control("A"       )       )
					self.assertFalse(_is_control(" "       )       )
					self.assertFalse(_is_control("\t"       )       )
					self.assertFalse(_is_control("\r"       )       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					self.assertTrue(_is_punctuation("-"       )       )
					self.assertTrue(_is_punctuation("$"       )       )
					self.assertTrue(_is_punctuation("`"       )       )
					self.assertTrue(_is_punctuation("."       )       )

					self.assertFalse(_is_punctuation("A"       )       )
					self.assertFalse(_is_punctuation(" "       )       )






	def    lowerCAmelCase_							(		self       )						->    Dict:
					snake_case__      :Optional[int]         =   self.get_tokenizer()
					snake_case__      :Any         =   self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(UpperCamelCase       ) for t in ["Test", "\xad", "test"]]					,[["[UNK]"], [], ["[UNK]"]]       )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(UpperCamelCase       ) for t in ["Test", "\xad", "test"]]					,[["[UNK]"], [], ["[UNK]"]]       )






	@slow
	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :List[str]         =   self.tokenizer_class.from_pretrained("bert-base-uncased"       )

					snake_case__      :Tuple         =   tokenizer.encode("sequence builders"					,add_special_tokens=UpperCamelCase       )
					snake_case__      :int         =   tokenizer.encode("multi-sequence build"					,add_special_tokens=UpperCamelCase       )

					snake_case__      :Tuple         =   tokenizer.build_inputs_with_special_tokens(UpperCamelCase       )
					snake_case__      :Union[str, Any]         =   tokenizer.build_inputs_with_special_tokens(UpperCamelCase					,UpperCamelCase       )

					assert encoded_sentence == [101] + text + [102]
					assert encoded_pair == [101] + text + [102] + text_a + [102]






	def    lowerCAmelCase_							(		self       )						->    Dict:





					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
													snake_case__      :int         =   self.rust_tokenizer_class.from_pretrained(UpperCamelCase					,**UpperCamelCase       )

													snake_case__      :Optional[int]         =   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													snake_case__      :Union[str, Any]         =   tokenizer_r.encode_plus(
													    UpperCamelCase					,return_attention_mask=UpperCamelCase					,return_token_type_ids=UpperCamelCase					,return_offsets_mapping=UpperCamelCase					,add_special_tokens=UpperCamelCase					,)

													snake_case__      :List[Any]         =   tokenizer_r.do_lower_case if hasattr(UpperCamelCase					,"do_lower_case"       ) else False
													snake_case__      :List[Any]         =   (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'A'),
													        ((1, 2), ','),
													        ((3, 5), 'na'),
													        ((5, 6), '##ï'),
													        ((6, 8), '##ve'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'Allen'),
													        ((21, 23), '##NL'),
													        ((23, 24), '##P'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'a'),
													        ((1, 2), ','),
													        ((3, 8), 'naive'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'allen'),
													        ((21, 23), '##nl'),
													        ((23, 24), '##p'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]					,tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
													self.assertEqual([e[0] for e in expected_results]					,tokens["offset_mapping"]       )






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :Optional[Any]         =   ['的', '人', '有']
					snake_case__      :int         =   ''.join(UpperCamelCase       )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
													snake_case__      :int         =   True
													snake_case__      :List[str]         =   self.tokenizer_class.from_pretrained(UpperCamelCase					,**UpperCamelCase       )
													snake_case__      :Tuple         =   self.rust_tokenizer_class.from_pretrained(UpperCamelCase					,**UpperCamelCase       )

													snake_case__      :Any         =   tokenizer_p.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )
													snake_case__      :int         =   tokenizer_r.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )

													snake_case__      :Optional[Any]         =   tokenizer_r.convert_ids_to_tokens(UpperCamelCase       )
													snake_case__      :Any         =   tokenizer_p.convert_ids_to_tokens(UpperCamelCase       )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(UpperCamelCase					,UpperCamelCase       )
													self.assertListEqual(UpperCamelCase					,UpperCamelCase       )

													snake_case__      :str         =   False
													snake_case__      :Union[str, Any]         =   self.rust_tokenizer_class.from_pretrained(UpperCamelCase					,**UpperCamelCase       )
													snake_case__      :List[Any]         =   self.tokenizer_class.from_pretrained(UpperCamelCase					,**UpperCamelCase       )

													snake_case__      :Union[str, Any]         =   tokenizer_r.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )
													snake_case__      :Tuple         =   tokenizer_p.encode(UpperCamelCase					,add_special_tokens=UpperCamelCase       )

													snake_case__      :Any         =   tokenizer_r.convert_ids_to_tokens(UpperCamelCase       )
													snake_case__      :Optional[int]         =   tokenizer_p.convert_ids_to_tokens(UpperCamelCase       )

													# it is expected that only the first Chinese character is not preceded by "##".
													snake_case__      :Dict         =   [
													    f'##{token}' if idx != 0 else token for idx, token in enumerate(UpperCamelCase       )
													]
													self.assertListEqual(UpperCamelCase					,UpperCamelCase       )
													self.assertListEqual(UpperCamelCase					,UpperCamelCase       )







import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
	from cookiecutter.main import cookiecutter

	__UpperCAmelCase							:   Dict							=  True
except ImportError:
	__UpperCAmelCase							:   List[Any]							=  False

__UpperCAmelCase							:   Dict							=  logging.get_logger(__name__)  # pylint: disable=invalid-name





def 					lowercase_ (    __snake_case     :							Namespace      )     -> Dict:





				'''simple docstring'''


				return AddNewModelCommand(args.testing	,	args.testing_file	,	path=args.path      )


class 					_snake_case						(			_A ):






	@staticmethod
	def    lowerCAmelCase_							(		UpperCamelCase       )						->    Any:
					snake_case__      :Dict         =   parser.add_parser("add-new-model"       )
					add_new_model_parser.add_argument("--testing"					,action="store_true"					,help="If in testing mode."       )
					add_new_model_parser.add_argument("--testing_file"					,type=UpperCamelCase					,help="Configuration file on which to run."       )
					add_new_model_parser.add_argument(
					    "--path"					,type=UpperCamelCase					,help="Path to cookiecutter. Should only be used for testing purposes."       )
					add_new_model_parser.set_defaults(func=UpperCamelCase       )






	def __init__(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase=None					,*UpperCamelCase       )						->    Any:
					snake_case__      :Union[str, Any]         =   testing
					snake_case__      :Union[str, Any]         =   testing_file
					snake_case__      :List[str]         =   path






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					warnings.warn(
					    "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
					    "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
					    "checks, you should use `transformers-cli add-new-model-like` instead."       )
					if not _has_cookiecutter:
									raise ImportError(
									    "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
									    "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"       )
					# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
					snake_case__      :Tuple         =   [directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
					if len(UpperCamelCase       ) > 0:
									raise ValueError(
									    "Several directories starting with `cookiecutter-template-` in current working directory. "
									    "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
									    "change your working directory."       )

					snake_case__      :str         =   (
					    Path(UpperCamelCase       ).parent.parent.parent.parent if self._path is None else Path(self._path       ).parent.parent
					)
					snake_case__      :Tuple         =   path_to_transformer_root / "templates" / "adding_a_new_model"

					# Execute cookiecutter
					if not self._testing:
									cookiecutter(str(UpperCamelCase       )       )
					else:
									with open(self._testing_file					,"r"       ) as configuration_file:
													snake_case__      :str         =   json.load(UpperCamelCase       )

									cookiecutter(
									    str(path_to_cookiecutter if self._path is None else self._path       )					,no_input=UpperCamelCase					,extra_context=UpperCamelCase					,)

					snake_case__      :List[Any]         =   [directory for directory in os.listdir() if "cookiecutter-template-" in directory[:22]][0]

					# Retrieve configuration
					with open(directory + "/configuration.json"					,"r"       ) as configuration_file:
									snake_case__      :Dict         =   json.load(UpperCamelCase       )

					snake_case__      :Optional[Any]         =   configuration["lowercase_modelname"]
					snake_case__      :List[Any]         =   configuration["generate_tensorflow_pytorch_and_flax"]
					os.remove(f'{directory}/configuration.json'       )

					snake_case__      :Any         =   "PyTorch" in generate_tensorflow_pytorch_and_flax
					snake_case__      :Any         =   "TensorFlow" in generate_tensorflow_pytorch_and_flax
					snake_case__      :Any         =   "Flax" in generate_tensorflow_pytorch_and_flax

					snake_case__      :Dict         =   f'{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'
					os.makedirs(UpperCamelCase					,exist_ok=UpperCamelCase       )
					os.makedirs(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}'					,exist_ok=UpperCamelCase       )

					# Tests require submodules as they have parent imports
					with open(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'					,"w"       ):
									pass

					shutil.move(
					    f'{directory}/__init__.py'					,f'{model_dir}/__init__.py'					,)
					shutil.move(
					    f'{directory}/configuration_{lowercase_model_name}.py'					,f'{model_dir}/configuration_{lowercase_model_name}.py'					,)

					def remove_copy_lines(UpperCamelCase       ):
									with open(UpperCamelCase					,"r"       ) as f:
													snake_case__      :List[str]         =   f.readlines()
									with open(UpperCamelCase					,"w"       ) as f:
													for line in lines:
																	if "# Copied from transformers." not in line:
																					f.write(UpperCamelCase       )

					if output_pytorch:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_{lowercase_model_name}.py'					,f'{model_dir}/modeling_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_{lowercase_model_name}.py'       )

					if output_tensorflow:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_tf_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_tf_{lowercase_model_name}.py'					,f'{model_dir}/modeling_tf_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_tf_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_tf_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_tf_{lowercase_model_name}.py'       )

					if output_flax:
									if not self._testing:
													remove_copy_lines(f'{directory}/modeling_flax_{lowercase_model_name}.py'       )

									shutil.move(
									    f'{directory}/modeling_flax_{lowercase_model_name}.py'					,f'{model_dir}/modeling_flax_{lowercase_model_name}.py'					,)

									shutil.move(
									    f'{directory}/test_modeling_flax_{lowercase_model_name}.py'					,f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'					,)
					else:
									os.remove(f'{directory}/modeling_flax_{lowercase_model_name}.py'       )
									os.remove(f'{directory}/test_modeling_flax_{lowercase_model_name}.py'       )

					shutil.move(
					    f'{directory}/{lowercase_model_name}.md'					,f'{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'					,)

					shutil.move(
					    f'{directory}/tokenization_{lowercase_model_name}.py'					,f'{model_dir}/tokenization_{lowercase_model_name}.py'					,)

					shutil.move(
					    f'{directory}/tokenization_fast_{lowercase_model_name}.py'					,f'{model_dir}/tokenization_{lowercase_model_name}_fast.py'					,)

					from os import fdopen, remove
					from shutil import copymode, move
					from tempfile import mkstemp

					def replace(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       ):
									# Create temp file
									snake_case__    ,						snake_case__      :Optional[Any]         =   mkstemp()
									snake_case__      :Optional[Any]         =   False
									with fdopen(UpperCamelCase					,"w"       ) as new_file:
													with open(UpperCamelCase       ) as old_file:
																	for line in old_file:
																					new_file.write(UpperCamelCase       )
																					if line_to_copy_below in line:
																									snake_case__      :Optional[Any]         =   True
																									for line_to_copy in lines_to_copy:
																													new_file.write(UpperCamelCase       )

									if not line_found:
													raise ValueError(f'Line {line_to_copy_below} was not found in file.'       )

									# Copy the file permissions from the old file to the new file
									copymode(UpperCamelCase					,UpperCamelCase       )
									# Remove original file
									remove(UpperCamelCase       )
									# Move new file
									move(UpperCamelCase					,UpperCamelCase       )

					def skip_units(UpperCamelCase       ):
									return (
									    ("generating PyTorch" in line and not output_pytorch)
									    or ("generating TensorFlow" in line and not output_tensorflow)
									    or ("generating Flax" in line and not output_flax)
									)

					def replace_in_files(UpperCamelCase       ):
									with open(UpperCamelCase       ) as datafile:
													snake_case__      :int         =   []
													snake_case__      :Optional[int]         =   False
													snake_case__      :List[str]         =   False
													for line in datafile:
																	if "# To replace in: " in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   line.split("\""       )[1]
																					snake_case__      :Tuple         =   skip_units(UpperCamelCase       )
																	elif "# Below: " in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   line.split("\""       )[1]
																					snake_case__      :List[str]         =   skip_units(UpperCamelCase       )
																	elif "# End." in line and "##" not in line:
																					if not skip_file and not skip_snippet:
																									replace(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )

																					snake_case__      :Tuple         =   []
																	elif "# Replace with" in line and "##" not in line:
																					snake_case__      :Optional[Any]         =   []
																	elif "##" not in line:
																					lines_to_copy.append(UpperCamelCase       )

									remove(UpperCamelCase       )

					replace_in_files(f'{directory}/to_replace_{lowercase_model_name}.py'       )
					os.rmdir(UpperCamelCase       )