
import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


lowerCamelCase__    :		Optional[Any]          =   """hf-internal-testing/tiny-random-bert"""
lowerCamelCase__    :		Any          =   os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
lowerCamelCase__    :		Any          =   """9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""



class       _snake_case   (	unittest.TestCase							):


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   str    		=      cached_file(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   # Should have downloaded the file in here
   self.assertTrue(os.path.isdir(SCREAMING_SNAKE_CASE_))
   # Cache should contain at least those three subfolders:
   for subfolder in ["blobs", "refs", "snapshots"]:
     self.assertTrue(os.path.isdir(os.path.join(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)))
   with open(os.path.join(SCREAMING_SNAKE_CASE_					,  """refs"""					,  """main""")) as f:
     lowercase__  :   List[str]    		=      f.read()
   self.assertEqual(SCREAMING_SNAKE_CASE_					,  os.path.join(SCREAMING_SNAKE_CASE_					,  """snapshots"""					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_))
   self.assertTrue(os.path.isfile(SCREAMING_SNAKE_CASE_))

   # File is cached at the same place the second time.
   lowercase__  :   Any    		=      cached_file(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   self.assertEqual(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

   # Using a specific revision to test the full commit hash.
   lowercase__  :   int    		=      cached_file(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  revision="""9b8c223""")
   self.assertEqual(SCREAMING_SNAKE_CASE_					,  os.path.join(SCREAMING_SNAKE_CASE_					,  """snapshots"""					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_))


 def      lowercase__     (					self):


   '''simple docstring'''




   with self.assertRaisesRegex(SCREAMING_SNAKE_CASE_					,  """is not a valid model identifier"""):
     lowercase__  :   Dict    		=      cached_file("""tiny-random-bert"""					,  SCREAMING_SNAKE_CASE_)

   with self.assertRaisesRegex(SCREAMING_SNAKE_CASE_					,  """is not a valid git identifier"""):
     lowercase__  :   List[str]    		=      cached_file(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  revision="""aaaa""")

   with self.assertRaisesRegex(SCREAMING_SNAKE_CASE_					,  """does not appear to have a file named"""):
     lowercase__  :   int    		=      cached_file(SCREAMING_SNAKE_CASE_					,  """conf""")


 def      lowercase__     (					self):


   '''simple docstring'''




   with self.assertRaisesRegex(SCREAMING_SNAKE_CASE_					,  """does not appear to have a file named"""):
     lowercase__  :   Dict    		=      cached_file(SCREAMING_SNAKE_CASE_					,  """conf""")

   with open(os.path.join(SCREAMING_SNAKE_CASE_					,  """refs"""					,  """main""")) as f:
     lowercase__  :   List[Any]    		=      f.read()
   self.assertTrue(os.path.isfile(os.path.join(SCREAMING_SNAKE_CASE_					,  """.no_exist"""					,  SCREAMING_SNAKE_CASE_					,  """conf""")))

   lowercase__  :   Optional[Any]    		=      cached_file(SCREAMING_SNAKE_CASE_					,  """conf"""					,  _raise_exceptions_for_missing_entries=SCREAMING_SNAKE_CASE_)
   self.assertIsNone(SCREAMING_SNAKE_CASE_)

   lowercase__  :   str    		=      cached_file(SCREAMING_SNAKE_CASE_					,  """conf"""					,  local_files_only=SCREAMING_SNAKE_CASE_					,  _raise_exceptions_for_missing_entries=SCREAMING_SNAKE_CASE_)
   self.assertIsNone(SCREAMING_SNAKE_CASE_)

   lowercase__  :   Dict    		=      mock.Mock()
   lowercase__  :   Optional[int]    		=      5_00
   lowercase__  :   List[Any]    		=      {}
   lowercase__  :   Any    		=      HTTPError
   lowercase__  :   Tuple    		=      {}

   # Under the mock environment we get a 500 error when trying to reach the tokenizer.
   with mock.patch("""requests.Session.request"""					,  return_value=SCREAMING_SNAKE_CASE_) as mock_head:
     lowercase__  :   Any    		=      cached_file(SCREAMING_SNAKE_CASE_					,  """conf"""					,  _raise_exceptions_for_connection_errors=SCREAMING_SNAKE_CASE_)
     self.assertIsNone(SCREAMING_SNAKE_CASE_)
     # This check we did call the fake head request
     mock_head.assert_called()


 def      lowercase__     (					self):


   '''simple docstring'''




   self.assertTrue(has_file("""hf-internal-testing/tiny-bert-pt-only"""					,  SCREAMING_SNAKE_CASE_))
   self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""					,  SCREAMING_SNAKE_CASE_))
   self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""					,  SCREAMING_SNAKE_CASE_))


 def      lowercase__     (					self):


   '''simple docstring'''




   self.assertIsNone(get_file_from_repo("""bert-base-cased"""					,  """ahah.txt"""))

   # The function raises if the repository does not exist.
   with self.assertRaisesRegex(SCREAMING_SNAKE_CASE_					,  """is not a valid model identifier"""):
     get_file_from_repo("""bert-base-case"""					,  SCREAMING_SNAKE_CASE_)

   # The function raises if the revision does not exist.
   with self.assertRaisesRegex(SCREAMING_SNAKE_CASE_					,  """is not a valid git identifier"""):
     get_file_from_repo("""bert-base-cased"""					,  SCREAMING_SNAKE_CASE_					,  revision="""ahaha""")

   lowercase__  :   Dict    		=      get_file_from_repo("""bert-base-cased"""					,  SCREAMING_SNAKE_CASE_)
   # The name is the cached name which is not very easy to test, so instead we load the content.
   lowercase__  :   int    		=      json.loads(open(SCREAMING_SNAKE_CASE_					,  """r""").read())
   self.assertEqual(config["""hidden_size"""]					,  7_68)



 def      lowercase__     (					self):


   '''simple docstring'''




   with tempfile.TemporaryDirectory() as tmp_dir:
     lowercase__  :   Tuple    		=      Path(SCREAMING_SNAKE_CASE_) / """a.txt"""
     filename.touch()
     self.assertEqual(get_file_from_repo(SCREAMING_SNAKE_CASE_					,  """a.txt""")					,  str(SCREAMING_SNAKE_CASE_))

     self.assertIsNone(get_file_from_repo(SCREAMING_SNAKE_CASE_					,  """b.txt"""))








import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow



class       _snake_case   :


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return None



class       _snake_case   :


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return None



class       _snake_case   (	unittest.TestCase							):
 __lowerCAmelCase  :     Optional[Any]        =			[
     # (model_name, model_kwargs)
     ('bert-base-cased', {}),
     ('gpt2', {'use_cache': False}),  # We don't support exporting GPT2 past keys anymore
 ]


 @require_tf
 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
     self._test_export(SCREAMING_SNAKE_CASE_					,  """tf"""					,  12					,  **SCREAMING_SNAKE_CASE_)


 @require_torch
 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
     self._test_export(SCREAMING_SNAKE_CASE_					,  """pt"""					,  12					,  **SCREAMING_SNAKE_CASE_)


 @require_torch
 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   from transformers import BertModel

   lowercase__  :   Union[str, Any]    		=      ["""[UNK]""", """[SEP]""", """[CLS]""", """[PAD]""", """[MASK]""", """some""", """other""", """words"""]
   with NamedTemporaryFile(mode="""w+t""") as vocab_file:
     vocab_file.write("""\n""".join(SCREAMING_SNAKE_CASE_))
     vocab_file.flush()
     lowercase__  :   Optional[int]    		=      BertTokenizerFast(vocab_file.name)

   with TemporaryDirectory() as bert_save_dir:
     lowercase__  :   Union[str, Any]    		=      BertModel(BertConfig(vocab_size=len(SCREAMING_SNAKE_CASE_)))
     model.save_pretrained(SCREAMING_SNAKE_CASE_)
     self._test_export(SCREAMING_SNAKE_CASE_					,  """pt"""					,  12					,  SCREAMING_SNAKE_CASE_)


 @require_tf
 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
     lowercase__  :   Any    		=      self._test_export(SCREAMING_SNAKE_CASE_					,  """tf"""					,  12					,  **SCREAMING_SNAKE_CASE_)
     lowercase__  :   Optional[Any]    		=      quantize(Path(SCREAMING_SNAKE_CASE_))

     # Ensure the actual quantized model is not bigger than the original one
     if quantized_path.stat().st_size >= Path(SCREAMING_SNAKE_CASE_).stat().st_size:
       self.fail("""Quantized model is bigger than initial ONNX model""")


 @require_torch
 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
     lowercase__  :   Tuple    		=      self._test_export(SCREAMING_SNAKE_CASE_					,  """pt"""					,  12					,  **SCREAMING_SNAKE_CASE_)
     lowercase__  :   Dict    		=      quantize(SCREAMING_SNAKE_CASE_)

     # Ensure the actual quantized model is not bigger than the original one
     if quantized_path.stat().st_size >= Path(SCREAMING_SNAKE_CASE_).stat().st_size:
       self.fail("""Quantized model is bigger than initial ONNX model""")


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=None					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   try:
     # Compute path
     with TemporaryDirectory() as tempdir:
       lowercase__  :   List[Any]    		=      Path(SCREAMING_SNAKE_CASE_).joinpath("""model.onnx""")

     # Remove folder if exists
     if path.parent.exists():
       path.parent.rmdir()

     # Export
     convert(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_)

     return path
   except Exception as e:
     self.fail(SCREAMING_SNAKE_CASE_)


 @require_torch
 @require_tokenizers
 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   from transformers import BertModel

   lowercase__  :   Dict    		=      BertModel(BertConfig.from_pretrained("""lysandre/tiny-bert-random"""))
   lowercase__  :   Union[str, Any]    		=      BertTokenizerFast.from_pretrained("""lysandre/tiny-bert-random""")
   self._test_infer_dynamic_axis(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  """pt""")


 @require_tf
 @require_tokenizers
 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   from transformers import TFBertModel

   lowercase__  :   Union[str, Any]    		=      TFBertModel(BertConfig.from_pretrained("""lysandre/tiny-bert-random"""))
   lowercase__  :   Optional[int]    		=      BertTokenizerFast.from_pretrained("""lysandre/tiny-bert-random""")
   self._test_infer_dynamic_axis(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  """tf""")


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   Dict    		=      FeatureExtractionPipeline(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

   lowercase__  :   Union[str, Any]    		=      ["""input_ids""", """token_type_ids""", """attention_mask""", """output_0""", """output_1"""]
   lowercase__		,    lowercase__		,    lowercase__		,    lowercase__  :   List[str]    		=      infer_shapes(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

   # Assert all variables are present
   self.assertEqual(len(SCREAMING_SNAKE_CASE_)					,  len(SCREAMING_SNAKE_CASE_))
   self.assertTrue(all(var_name in shapes for var_name in variable_names))
   self.assertSequenceEqual(variable_names[:3]					,  SCREAMING_SNAKE_CASE_)
   self.assertSequenceEqual(variable_names[3:]					,  SCREAMING_SNAKE_CASE_)

   # Assert inputs are {0: batch, 1: sequence}
   for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
     self.assertDictEqual(shapes[var_name]					,  {0: """batch""", 1: """sequence"""})

   # Assert outputs are {0: batch, 1: sequence} and {0: batch}
   self.assertDictEqual(shapes["""output_0"""]					,  {0: """batch""", 1: """sequence"""})
   self.assertDictEqual(shapes["""output_1"""]					,  {0: """batch"""})


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Any    		=      ["""input_ids""", """attention_mask""", """token_type_ids"""]
   lowercase__  :   Tuple    		=      {"""input_ids""": [1, 2, 3, 4], """attention_mask""": [0, 0, 0, 0], """token_type_ids""": [1, 1, 1, 1]}
   lowercase__		,    lowercase__  :   Optional[Any]    		=      ensure_valid_input(FuncContiguousArgs()					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

   # Should have exactly the same number of args (all are valid)
   self.assertEqual(len(SCREAMING_SNAKE_CASE_)					,  3)

   # Should have exactly the same input names
   self.assertEqual(set(SCREAMING_SNAKE_CASE_)					,  set(SCREAMING_SNAKE_CASE_))

   # Parameter should be reordered according to their respective place in the function:
   # (input_ids, token_type_ids, attention_mask)
   self.assertEqual(SCREAMING_SNAKE_CASE_					,  (tokens["""input_ids"""], tokens["""token_type_ids"""], tokens["""attention_mask"""]))

   # Generated args are interleaved with another args (for instance parameter "past" in GPT2)
   lowercase__		,    lowercase__  :   Any    		=      ensure_valid_input(FuncNonContiguousArgs()					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

   # Should have exactly the one arg (all before the one not provided "some_other_args")
   self.assertEqual(len(SCREAMING_SNAKE_CASE_)					,  1)
   self.assertEqual(len(SCREAMING_SNAKE_CASE_)					,  1)

   # Should have only "input_ids"
   self.assertEqual(inputs_args[0]					,  tokens["""input_ids"""])
   self.assertEqual(ordered_input_names[0]					,  """input_ids""")



 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   List[str]    		=      generate_identified_filename(Path("""/home/something/my_fake_model.onnx""")					,  """-test""")
   self.assertEqual("""/home/something/my_fake_model-test.onnx"""					,  generated.as_posix())






