


"""simple docstring"""


import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
    from cookiecutter.main import cookiecutter

    __lowerCamelCase      :int	    =		True
except ImportError:
    __lowerCamelCase      :Tuple	    =		False

__lowerCamelCase      :Optional[int]	    =		logging.get_logger(__name__)  # pylint: disable=invalid-name




def   snake_case		(     UpperCamelCase__       :						Namespace    )   ->      Optional[int]:
      return AddNewModelCommand(args.testing			,				args.testing_file			,				path=args.path    )






class 			A__	(						__lowercase):


     """simple docstring"""

     @staticmethod
     def 		a__						(	__a:						ArgumentParser	)->    Tuple:
           lowerCamelCase					:					int        =	parser.add_parser("""add-new-model"""	)
           add_new_model_parser.add_argument("""--testing"""     ,		action="""store_true"""     ,		help="""If in testing mode."""	)
           add_new_model_parser.add_argument("""--testing_file"""     ,		type=__a     ,		help="""Configuration file on which to run."""	)
           add_new_model_parser.add_argument(
               """--path"""     ,		type=__a     ,		help="""Path to cookiecutter. Should only be used for testing purposes."""	)
           add_new_model_parser.set_defaults(func=__a	)

     def __init__(	self:						List[Any]     ,		__a:						bool     ,		__a:						str     ,		__a:						Dict=None     ,		*__a:						Optional[Any]	)->    int:
           lowerCamelCase					:					Any        =	testing
           lowerCamelCase					:					Any        =	testing_file
           lowerCamelCase					:					Optional[Any]        =	path







     def 		a__						(	self:						int	)->    Optional[int]:
           warnings.warn(
               """The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. """
               """It is not actively maintained anymore, so might give a result that won't pass all tests and quality """
               """checks, you should use `transformers-cli add-new-model-like` instead."""	)
           if not _has_cookiecutter:
                 raise ImportError(
                     """Model creation dependencies are required to use the `add_new_model` command. Install them by running """
                     """the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"""	)
           # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
           lowerCamelCase					:					Optional[Any]        =	[directory for directory in os.listdir() if """cookiecutter-template-""" == directory[:22]]
           if len(__a	) > 0:
                 raise ValueError(
                     """Several directories starting with `cookiecutter-template-` in current working directory. """
                     """Please clean your directory by removing all folders starting with `cookiecutter-template-` or """
                     """change your working directory."""	)

           lowerCamelCase					:					List[Any]        =	(
               Path(__a	).parent.parent.parent.parent if self._path is None else Path(self._path	).parent.parent
           )
           lowerCamelCase					:					Optional[int]        =	path_to_transformer_root / """templates""" / """adding_a_new_model"""

           # Execute cookiecutter
           if not self._testing:
                 cookiecutter(str(__a	)	)
           else:
                 with open(self._testing_file     ,		"""r"""	) as configuration_file:
                       lowerCamelCase					:					Union[str, Any]        =	json.load(__a	)

                 cookiecutter(
                     str(path_to_cookiecutter if self._path is None else self._path	)     ,		no_input=__a     ,		extra_context=__a     ,		)

           lowerCamelCase					:					Dict        =	[directory for directory in os.listdir() if """cookiecutter-template-""" in directory[:22]][0]

           # Retrieve configuration
           with open(directory + """/configuration.json"""     ,		"""r"""	) as configuration_file:
                 lowerCamelCase					:					List[str]        =	json.load(__a	)

           lowerCamelCase					:					Union[str, Any]        =	configuration["""lowercase_modelname"""]
           lowerCamelCase					:					List[str]        =	configuration["""generate_tensorflow_pytorch_and_flax"""]
           os.remove(f'{directory}/configuration.json'	)

           lowerCamelCase					:					Optional[int]        =	"""PyTorch""" in generate_tensorflow_pytorch_and_flax
           lowerCamelCase					:					List[str]        =	"""TensorFlow""" in generate_tensorflow_pytorch_and_flax
           lowerCamelCase					:					Union[str, Any]        =	"""Flax""" in generate_tensorflow_pytorch_and_flax

           lowerCamelCase					:					List[Any]        =	f'{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'
           os.makedirs(__a     ,		exist_ok=__a	)
           os.makedirs(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}'     ,		exist_ok=__a	)

           # Tests require submodules as they have parent imports
           with open(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'     ,		"""w"""	):
                 pass

           shutil.move(
               f'{directory}/__init__.py'     ,		f'{model_dir}/__init__.py'     ,		)
           shutil.move(
               f'{directory}/configuration_{lowercase_model_name}.py'     ,		f'{model_dir}/configuration_{lowercase_model_name}.py'     ,		)

           def remove_copy_lines(__a:						int	):
                 with open(__a     ,		"""r"""	) as f:
                       lowerCamelCase					:					List[str]        =	f.readlines()
                 with open(__a     ,		"""w"""	) as f:
                       for line in lines:
                             if "# Copied from transformers." not in line:
                                   f.write(__a	)

           if output_pytorch:
                 if not self._testing:
                       remove_copy_lines(f'{directory}/modeling_{lowercase_model_name}.py'	)

                 shutil.move(
                     f'{directory}/modeling_{lowercase_model_name}.py'     ,		f'{model_dir}/modeling_{lowercase_model_name}.py'     ,		)

                 shutil.move(
                     f'{directory}/test_modeling_{lowercase_model_name}.py'     ,		f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'     ,		)
           else:
                 os.remove(f'{directory}/modeling_{lowercase_model_name}.py'	)
                 os.remove(f'{directory}/test_modeling_{lowercase_model_name}.py'	)

           if output_tensorflow:
                 if not self._testing:
                       remove_copy_lines(f'{directory}/modeling_tf_{lowercase_model_name}.py'	)

                 shutil.move(
                     f'{directory}/modeling_tf_{lowercase_model_name}.py'     ,		f'{model_dir}/modeling_tf_{lowercase_model_name}.py'     ,		)

                 shutil.move(
                     f'{directory}/test_modeling_tf_{lowercase_model_name}.py'     ,		f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'     ,		)
           else:
                 os.remove(f'{directory}/modeling_tf_{lowercase_model_name}.py'	)
                 os.remove(f'{directory}/test_modeling_tf_{lowercase_model_name}.py'	)

           if output_flax:
                 if not self._testing:
                       remove_copy_lines(f'{directory}/modeling_flax_{lowercase_model_name}.py'	)

                 shutil.move(
                     f'{directory}/modeling_flax_{lowercase_model_name}.py'     ,		f'{model_dir}/modeling_flax_{lowercase_model_name}.py'     ,		)

                 shutil.move(
                     f'{directory}/test_modeling_flax_{lowercase_model_name}.py'     ,		f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'     ,		)
           else:
                 os.remove(f'{directory}/modeling_flax_{lowercase_model_name}.py'	)
                 os.remove(f'{directory}/test_modeling_flax_{lowercase_model_name}.py'	)

           shutil.move(
               f'{directory}/{lowercase_model_name}.md'     ,		f'{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'     ,		)

           shutil.move(
               f'{directory}/tokenization_{lowercase_model_name}.py'     ,		f'{model_dir}/tokenization_{lowercase_model_name}.py'     ,		)

           shutil.move(
               f'{directory}/tokenization_fast_{lowercase_model_name}.py'     ,		f'{model_dir}/tokenization_{lowercase_model_name}_fast.py'     ,		)

           from os import fdopen, remove
           from shutil import copymode, move
           from tempfile import mkstemp

           def replace(__a:						str     ,		__a:						str     ,		__a:						List[str]	):
                 # Create temp file
                 lowerCamelCase					:					List[Any]        =	mkstemp()
                 lowerCamelCase					:					Dict        =	False
                 with fdopen(__a     ,		"""w"""	) as new_file:
                       with open(__a	) as old_file:
                             for line in old_file:
                                   new_file.write(__a	)
                                   if line_to_copy_below in line:
                                         lowerCamelCase					:					List[str]        =	True
                                         for line_to_copy in lines_to_copy:
                                               new_file.write(__a	)

                 if not line_found:
                       raise ValueError(f'Line {line_to_copy_below} was not found in file.'	)

                 # Copy the file permissions from the old file to the new file
                 copymode(__a     ,		__a	)
                 # Remove original file
                 remove(__a	)
                 # Move new file
                 move(__a     ,		__a	)

           def skip_units(__a:						Union[str, Any]	):
                 return (
                     ("generating PyTorch" in line and not output_pytorch)
                     or ("generating TensorFlow" in line and not output_tensorflow)
                     or ("generating Flax" in line and not output_flax)
                 )

           def replace_in_files(__a:						Union[str, Any]	):
                 with open(__a	) as datafile:
                       lowerCamelCase					:					Tuple        =	[]
                       lowerCamelCase					:					Dict        =	False
                       lowerCamelCase					:					Optional[int]        =	False
                       for line in datafile:
                             if "# To replace in: " in line and "##" not in line:
                                   lowerCamelCase					:					Union[str, Any]        =	line.split("""\""""	)[1]
                                   lowerCamelCase					:					Any        =	skip_units(__a	)
                             elif "# Below: " in line and "##" not in line:
                                   lowerCamelCase					:					List[str]        =	line.split("""\""""	)[1]
                                   lowerCamelCase					:					List[str]        =	skip_units(__a	)
                             elif "# End." in line and "##" not in line:
                                   if not skip_file and not skip_snippet:
                                         replace(__a     ,		__a     ,		__a	)

                                   lowerCamelCase					:					str        =	[]
                             elif "# Replace with" in line and "##" not in line:
                                   lowerCamelCase					:					Dict        =	[]
                             elif "##" not in line:
                                   lines_to_copy.append(__a	)

                 remove(__a	)

           replace_in_files(f'{directory}/to_replace_{lowercase_model_name}.py'	)
           os.rmdir(__a	)






"""simple docstring"""


import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow


class 			A__	:


     """simple docstring"""

     def 		a__						(	self:						Optional[int]     ,		__a:						Optional[int]     ,		__a:						Tuple     ,		__a:						Optional[int]	)->    List[str]:
           return None


class 			A__	:


     """simple docstring"""

     def 		a__						(	self:						Optional[int]     ,		__a:						Tuple     ,		__a:						str     ,		__a:						str     ,		__a:						str	)->    Tuple:
           return None


class 			A__	(						unittest.TestCase):


     """simple docstring"""
     snake_case__	:					Optional[Any]													=[
         # (model_name, model_kwargs)
         ('''bert-base-cased''', {}),
         ('''gpt2''', {'''use_cache''': False}),  # We don't support exporting GPT2 past keys anymore
     ]

     @require_tf
     @slow
     def 		a__						(	self:						Optional[Any]	)->    int:
           for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
                 self._test_export(__a     ,		"""tf"""     ,		12     ,		**__a	)

     @require_torch
     @slow
     def 		a__						(	self:						str	)->    int:
           for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
                 self._test_export(__a     ,		"""pt"""     ,		12     ,		**__a	)

     @require_torch
     @slow
     def 		a__						(	self:						Union[str, Any]	)->    Dict:
           from transformers import BertModel

           lowerCamelCase					:					int        =	["""[UNK]""", """[SEP]""", """[CLS]""", """[PAD]""", """[MASK]""", """some""", """other""", """words"""]
           with NamedTemporaryFile(mode="""w+t"""	) as vocab_file:
                 vocab_file.write("""\n""".join(__a	)	)
                 vocab_file.flush()
                 lowerCamelCase					:					Dict        =	BertTokenizerFast(vocab_file.name	)

           with TemporaryDirectory() as bert_save_dir:
                 lowerCamelCase					:					List[str]        =	BertModel(BertConfig(vocab_size=len(__a	)	)	)
                 model.save_pretrained(__a	)
                 self._test_export(__a     ,		"""pt"""     ,		12     ,		__a	)

     @require_tf
     @slow
     def 		a__						(	self:						Optional[Any]	)->    Optional[int]:
           for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
                 lowerCamelCase					:					Optional[int]        =	self._test_export(__a     ,		"""tf"""     ,		12     ,		**__a	)
                 lowerCamelCase					:					Tuple        =	quantize(Path(__a	)	)

                 # Ensure the actual quantized model is not bigger than the original one
                 if quantized_path.stat().st_size >= Path(__a	).stat().st_size:
                       self.fail("""Quantized model is bigger than initial ONNX model"""	)

     @require_torch
     @slow
     def 		a__						(	self:						Any	)->    Optional[int]:
           for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
                 lowerCamelCase					:					Any        =	self._test_export(__a     ,		"""pt"""     ,		12     ,		**__a	)
                 lowerCamelCase					:					Dict        =	quantize(__a	)

                 # Ensure the actual quantized model is not bigger than the original one
                 if quantized_path.stat().st_size >= Path(__a	).stat().st_size:
                       self.fail("""Quantized model is bigger than initial ONNX model"""	)

     def 		a__						(	self:						List[Any]     ,		__a:						Optional[Any]     ,		__a:						List[Any]     ,		__a:						Union[str, Any]     ,		__a:						Optional[Any]=None     ,		**__a:						Optional[int]	)->    Any:
           try:
                 # Compute path
                 with TemporaryDirectory() as tempdir:
                       lowerCamelCase					:					Optional[Any]        =	Path(__a	).joinpath("""model.onnx"""	)

                 # Remove folder if exists
                 if path.parent.exists():
                       path.parent.rmdir()

                 # Export
                 convert(__a     ,		__a     ,		__a     ,		__a     ,		__a     ,		**__a	)

                 return path
           except Exception as e:
                 self.fail(__a	)

     @require_torch
     @require_tokenizers
     @slow
     def 		a__						(	self:						Tuple	)->    Dict:
           from transformers import BertModel

           lowerCamelCase					:					int        =	BertModel(BertConfig.from_pretrained("""lysandre/tiny-bert-random"""	)	)
           lowerCamelCase					:					List[Any]        =	BertTokenizerFast.from_pretrained("""lysandre/tiny-bert-random"""	)
           self._test_infer_dynamic_axis(__a     ,		__a     ,		"""pt"""	)

     @require_tf
     @require_tokenizers
     @slow
     def 		a__						(	self:						Optional[Any]	)->    List[Any]:
           from transformers import TFBertModel

           lowerCamelCase					:					Union[str, Any]        =	TFBertModel(BertConfig.from_pretrained("""lysandre/tiny-bert-random"""	)	)
           lowerCamelCase					:					str        =	BertTokenizerFast.from_pretrained("""lysandre/tiny-bert-random"""	)
           self._test_infer_dynamic_axis(__a     ,		__a     ,		"""tf"""	)

     def 		a__						(	self:						List[str]     ,		__a:						str     ,		__a:						Optional[Any]     ,		__a:						str	)->    List[Any]:
           lowerCamelCase					:					List[str]        =	FeatureExtractionPipeline(__a     ,		__a	)

           lowerCamelCase					:					List[str]        =	["""input_ids""", """token_type_ids""", """attention_mask""", """output_0""", """output_1"""]
           lowerCamelCase     ,			lowerCamelCase     ,			lowerCamelCase     ,			lowerCamelCase					:					Tuple        =	infer_shapes(__a     ,		__a	)

           # Assert all variables are present
           self.assertEqual(len(__a	)     ,		len(__a	)	)
           self.assertTrue(all(var_name in shapes for var_name in variable_names	)	)
           self.assertSequenceEqual(variable_names[:3]     ,		__a	)
           self.assertSequenceEqual(variable_names[3:]     ,		__a	)

           # Assert inputs are {0: batch, 1: sequence}
           for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
                 self.assertDictEqual(shapes[var_name]     ,		{0: """batch""", 1: """sequence"""}	)

           # Assert outputs are {0: batch, 1: sequence} and {0: batch}
           self.assertDictEqual(shapes["""output_0"""]     ,		{0: """batch""", 1: """sequence"""}	)
           self.assertDictEqual(shapes["""output_1"""]     ,		{0: """batch"""}	)

     def 		a__						(	self:						List[Any]	)->    int:
           lowerCamelCase					:					List[str]        =	["""input_ids""", """attention_mask""", """token_type_ids"""]
           lowerCamelCase					:					str        =	{"""input_ids""": [1, 2, 3, 4], """attention_mask""": [0, 0, 0, 0], """token_type_ids""": [1, 1, 1, 1]}
           lowerCamelCase     ,			lowerCamelCase					:					List[Any]        =	ensure_valid_input(FuncContiguousArgs()     ,		__a     ,		__a	)

           # Should have exactly the same number of args (all are valid)
           self.assertEqual(len(__a	)     ,		3	)

           # Should have exactly the same input names
           self.assertEqual(set(__a	)     ,		set(__a	)	)

           # Parameter should be reordered according to their respective place in the function:
           # (input_ids, token_type_ids, attention_mask)
           self.assertEqual(__a     ,		(tokens["""input_ids"""], tokens["""token_type_ids"""], tokens["""attention_mask"""])	)

           # Generated args are interleaved with another args (for instance parameter "past" in GPT2)
           lowerCamelCase     ,			lowerCamelCase					:					List[Any]        =	ensure_valid_input(FuncNonContiguousArgs()     ,		__a     ,		__a	)

           # Should have exactly the one arg (all before the one not provided "some_other_args")
           self.assertEqual(len(__a	)     ,		1	)
           self.assertEqual(len(__a	)     ,		1	)

           # Should have only "input_ids"
           self.assertEqual(inputs_args[0]     ,		tokens["""input_ids"""]	)
           self.assertEqual(ordered_input_names[0]     ,		"""input_ids"""	)







     def 		a__						(	self:						Tuple	)->    Tuple:
           lowerCamelCase					:					Optional[int]        =	generate_identified_filename(Path("""/home/something/my_fake_model.onnx"""	)     ,		"""-test"""	)
           self.assertEqual("""/home/something/my_fake_model-test.onnx"""     ,		generated.as_posix()	)




