





import argparse
import os

import numpy as np
import tensorflow as tf
import torch

from transformers import BertModel






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							BertModel					,   UpperCamelCase__:							str					,   UpperCamelCase__:							str    ):
		SCREAMING_SNAKE_CASE__			=      ("""dense.weight""", """attention.self.query""", """attention.self.key""", """attention.self.value""")

		SCREAMING_SNAKE_CASE__			=      (
		    ("""layer.""", """layer_"""),
		    ("""word_embeddings.weight""", """word_embeddings"""),
		    ("""position_embeddings.weight""", """position_embeddings"""),
		    ("""token_type_embeddings.weight""", """token_type_embeddings"""),
		    (""".""", """/"""),
		    ("""LayerNorm/weight""", """LayerNorm/gamma"""),
		    ("""LayerNorm/bias""", """LayerNorm/beta"""),
		    ("""weight""", """kernel"""),
		)

		if not os.path.isdir(UpperCamelCase__    ):
				os.makedirs(UpperCamelCase__    )

		SCREAMING_SNAKE_CASE__			=      model.state_dict()

		def to_tf_var_name(UpperCamelCase__:							str    ):
				for patt, repl in iter(UpperCamelCase__    ):
						SCREAMING_SNAKE_CASE__			=      name.replace(UpperCamelCase__					,   UpperCamelCase__    )
				return f'''bert/{name}'''

		def create_tf_var(UpperCamelCase__:							np.ndarray					,   UpperCamelCase__:							str					,   UpperCamelCase__:							tf.Session    ):
				SCREAMING_SNAKE_CASE__			=      tf.dtypes.as_dtype(tensor.dtype    )
				SCREAMING_SNAKE_CASE__			=      tf.get_variable(dtype=UpperCamelCase__					,   shape=tensor.shape					,   name=UpperCamelCase__					,   initializer=tf.zeros_initializer()    )
				session.run(tf.variables_initializer([tf_var]    )    )
				session.run(UpperCamelCase__    )
				return tf_var

		tf.reset_default_graph()
		with tf.Session() as session:
				for var_name in state_dict:
						SCREAMING_SNAKE_CASE__			=      to_tf_var_name(UpperCamelCase__    )
						SCREAMING_SNAKE_CASE__			=      state_dict[var_name].numpy()
						if any(x in var_name for x in tensors_to_transpose    ):
								SCREAMING_SNAKE_CASE__			=      torch_tensor.T
						SCREAMING_SNAKE_CASE__			=      create_tf_var(tensor=UpperCamelCase__					,   name=UpperCamelCase__					,   session=UpperCamelCase__    )
						tf.keras.backend.set_value(UpperCamelCase__					,   UpperCamelCase__    )
						SCREAMING_SNAKE_CASE__			=      session.run(UpperCamelCase__    )
						print(f'''Successfully created {tf_name}: {np.allclose(UpperCamelCase__					,   UpperCamelCase__    )}'''    )

				SCREAMING_SNAKE_CASE__			=      tf.train.Saver(tf.trainable_variables()    )
				saver.save(UpperCamelCase__					,   os.path.join(UpperCamelCase__					,   model_name.replace("""-"""					,   """_"""    ) + """.ckpt"""    )    )






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[int]=None    ):
		SCREAMING_SNAKE_CASE__			=      argparse.ArgumentParser()
		parser.add_argument("""--model_name"""					,   type=UpperCamelCase__					,   required=UpperCamelCase__					,   help="""model name e.g. bert-base-uncased"""    )
		parser.add_argument(
		    """--cache_dir"""					,   type=UpperCamelCase__					,   default=UpperCamelCase__					,   required=UpperCamelCase__					,   help="""Directory containing pytorch model"""    )
		parser.add_argument("""--pytorch_model_path"""					,   type=UpperCamelCase__					,   required=UpperCamelCase__					,   help="""/path/to/<pytorch-model-name>.bin"""    )
		parser.add_argument("""--tf_cache_dir"""					,   type=UpperCamelCase__					,   required=UpperCamelCase__					,   help="""Directory in which to save tensorflow model"""    )
		SCREAMING_SNAKE_CASE__			=      parser.parse_args(UpperCamelCase__    )

		SCREAMING_SNAKE_CASE__			=      BertModel.from_pretrained(
		    pretrained_model_name_or_path=args.model_name					,   state_dict=torch.load(args.pytorch_model_path    )					,   cache_dir=args.cache_dir					,   )

		convert_pytorch_checkpoint_to_tf(model=UpperCamelCase__					,   ckpt_dir=args.tf_cache_dir					,   model_name=args.model_name    )


if __name__ == "__main__":
	main()











import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					RoCBertTokenizer
							lowerCamelCase_           =					None
							lowerCamelCase_           =					False
							lowerCamelCase_           =					True
							lowerCamelCase_           =					filter_non_english


							def      _snake_case  (							self    :List[Any]		)   ->   List[Any]:



									"""simple docstring"""



									super().setUp()

									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      {}
									for i, value in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_shape_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_pronunciation_file"""]		)
									with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
									with open(self.word_shape_file							,				"""w"""							,				encoding="""utf-8"""		) as word_shape_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)
									with open(self.word_pronunciation_file							,				"""w"""							,				encoding="""utf-8"""		) as word_pronunciation_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize("""你好[SEP]你是谁"""		)
									self.assertListEqual(__A							,				["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)


							def      _snake_case  (							self    :List[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""		)							,				["""ah""", """\u535A""", """\u63A8""", """zz"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""hello""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""h\u00E9llo"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :int		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :List[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				never_split=["""[UNK]"""]		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									SCREAMING_SNAKE_CASE__			=      {}
									for i, token in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      RoCBertWordpieceTokenizer(vocab=__A							,				unk_token="""[UNK]"""		)

									self.assertListEqual(tokenizer.tokenize(""""""		)							,				[]		)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""		)							,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]		)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""		)							,				["""[UNK]""", """runn""", """##ing"""]		)


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_whitespace(""" """		)		)
									self.assertTrue(_is_whitespace("""\t"""		)		)
									self.assertTrue(_is_whitespace("""\r"""		)		)
									self.assertTrue(_is_whitespace("""\n"""		)		)
									self.assertTrue(_is_whitespace("""\u00A0"""		)		)

									self.assertFalse(_is_whitespace("""A"""		)		)
									self.assertFalse(_is_whitespace("""-"""		)		)


							def      _snake_case  (							self    :int		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_control("""\u0005"""		)		)

									self.assertFalse(_is_control("""A"""		)		)
									self.assertFalse(_is_control(""" """		)		)
									self.assertFalse(_is_control("""\t"""		)		)
									self.assertFalse(_is_control("""\r"""		)		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									self.assertTrue(_is_punctuation("""-"""		)		)
									self.assertTrue(_is_punctuation("""$"""		)		)
									self.assertTrue(_is_punctuation("""`"""		)		)
									self.assertTrue(_is_punctuation("""."""		)		)

									self.assertFalse(_is_punctuation("""A"""		)		)
									self.assertFalse(_is_punctuation(""" """		)		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)

									if self.test_rust_tokenizer:
											SCREAMING_SNAKE_CASE__			=      self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)


							def      _snake_case  (							self    :int		)   ->   Any:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode_plus(
													    __A							,				return_attention_mask=__A							,				return_token_type_ids=__A							,				return_offsets_mapping=__A							,				add_special_tokens=__A							,				)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.do_lower_case if hasattr(__A							,				"""do_lower_case"""		) else False
													SCREAMING_SNAKE_CASE__			=      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """A"""),
													        ((1, 2), ""","""),
													        ((3, 5), """na"""),
													        ((5, 6), """##ï"""),
													        ((6, 8), """##ve"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """Allen"""),
													        ((21, 23), """##NL"""),
													        ((23, 24), """##P"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """a"""),
													        ((1, 2), ""","""),
													        ((3, 8), """naive"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """allen"""),
													        ((21, 23), """##nl"""),
													        ((23, 24), """##p"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]		)		)
													self.assertEqual([e[0] for e in expected_results]							,				tokens["""offset_mapping"""]		)


							def      _snake_case  (							self    :Optional[int]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""的""", """人""", """有"""]
									SCREAMING_SNAKE_CASE__			=      """""".join(__A		)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      True
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)

													SCREAMING_SNAKE_CASE__			=      False
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE__			=      [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__A		)
													]
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)


							@slow
							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你好"""							,				add_special_tokens=__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你是谁"""							,				add_special_tokens=__A		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A							,				__A		)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizers(do_lower_case=__A		)
									for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
													SCREAMING_SNAKE_CASE__			=      """你好，你是谁"""
													SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_shape_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_pronunciation_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.prepare_for_model(
													    __A							,				__A							,				__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer.encode_plus(__A							,				add_special_tokens=__A		)

													self.assertEqual(__A							,				__A		)