






import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


class        __UpperCamelCase    (     _lowerCAmelCase   ,			unittest.TestCase   ):
	__snake_case	:Dict   =       ProphetNetTokenizer
	__snake_case	:List[str]   =       False







	def  _a   (  self      :							Union[str, Any]			)		-> str:
		"""simple docstring"""







		super().setUp()

		__lowercase             =  [
		    """[UNK]""",
		    """[CLS]""",
		    """[SEP]""",
		    """[PAD]""",
		    """[MASK]""",
		    """want""",
		    """##want""",
		    """##ed""",
		    """wa""",
		    """un""",
		    """runn""",
		    """##ing""",
		    """,""",
		    """low""",
		    """lowest""",
		]
		__lowercase             =  os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""vocab_file"""]			)
		with open(self.vocab_file       ,				"""w"""       ,				encoding="""utf-8"""			) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)







	def  _a   (  self      :							Union[str, Any]       ,				_lowerCAmelCase      :							Optional[Any]			)		-> List[str]:
		"""simple docstring"""







		__lowercase             =  """UNwant\u00E9d,running"""
		__lowercase             =  """unwanted, running"""
		return input_text, output_text







	def  _a   (  self      :							Any			)		-> Optional[Any]:
		"""simple docstring"""







		__lowercase             =  self.tokenizer_class(self.vocab_file			)

		__lowercase             =  tokenizer.tokenize("""UNwant\u00E9d,running"""			)
		self.assertListEqual(_lowerCAmelCase       ,				["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]			)
		self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase			)       ,				[9, 6, 7, 12, 10, 11]			)







	def  _a   (  self      :							Union[str, Any]			)		-> Any:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer()

		self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""			)       ,				["""ah""", """\u535A""", """\u63A8""", """zz"""]			)







	def  _a   (  self      :							Tuple			)		-> Optional[Any]:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)       ,				["""hello""", """!""", """how""", """are""", """you""", """?"""]			)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)       ,				["""hello"""]			)







	def  _a   (  self      :							List[Any]			)		-> Union[str, Any]:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase       ,				strip_accents=_lowerCAmelCase			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)       ,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]			)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)       ,				["""h\u00E9llo"""]			)







	def  _a   (  self      :							List[str]			)		-> Optional[Any]:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase       ,				strip_accents=_lowerCAmelCase			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)       ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)       ,				["""hello"""]			)







	def  _a   (  self      :							Optional[int]			)		-> Union[str, Any]:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)       ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)       ,				["""hello"""]			)







	def  _a   (  self      :							List[str]			)		-> Dict:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)       ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)







	def  _a   (  self      :							int			)		-> Dict:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase       ,				strip_accents=_lowerCAmelCase			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)       ,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)







	def  _a   (  self      :							int			)		-> List[Any]:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase       ,				strip_accents=_lowerCAmelCase			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)       ,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)







	def  _a   (  self      :							List[Any]			)		-> str:
		"""simple docstring"""







		__lowercase             =  BasicTokenizer(do_lower_case=_lowerCAmelCase       ,				never_split=["""[UNK]"""]			)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""			)       ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]			)







	def  _a   (  self      :							Dict			)		-> List[Any]:
		"""simple docstring"""







		__lowercase             =  ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

		__lowercase             =  {}
		for i, token in enumerate(_lowerCAmelCase			):
			__lowercase             =  i
		__lowercase             =  WordpieceTokenizer(vocab=_lowerCAmelCase       ,				unk_token="""[UNK]"""			)

		self.assertListEqual(tokenizer.tokenize(""""""			)       ,				[]			)

		self.assertListEqual(tokenizer.tokenize("""unwanted running"""			)       ,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]			)

		self.assertListEqual(tokenizer.tokenize("""unwantedX running"""			)       ,				["""[UNK]""", """runn""", """##ing"""]			)







	@require_torch
	def  _a   (  self      :							List[Any]			)		-> Tuple:
		"""simple docstring"""







		__lowercase             =  self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""			)

		__lowercase             =  ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
		__lowercase             =  [1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
		__lowercase             =  tokenizer(_lowerCAmelCase       ,				padding=_lowerCAmelCase       ,				return_tensors="""pt"""			)
		self.assertIsInstance(_lowerCAmelCase       ,				_lowerCAmelCase			)
		__lowercase             =  list(batch.input_ids.numpy()[0]			)
		self.assertListEqual(_lowerCAmelCase       ,				_lowerCAmelCase			)

		self.assertEqual((2, 9)       ,				batch.input_ids.shape			)
		self.assertEqual((2, 9)       ,				batch.attention_mask.shape			)







	def  _a   (  self      :							Dict			)		-> List[Any]:
		"""simple docstring"""







		self.assertTrue(_is_whitespace(""" """			)			)
		self.assertTrue(_is_whitespace("""\t"""			)			)
		self.assertTrue(_is_whitespace("""\r"""			)			)
		self.assertTrue(_is_whitespace("""\n"""			)			)
		self.assertTrue(_is_whitespace("""\u00A0"""			)			)

		self.assertFalse(_is_whitespace("""A"""			)			)
		self.assertFalse(_is_whitespace("""-"""			)			)







	def  _a   (  self      :							Any			)		-> List[Any]:
		"""simple docstring"""







		self.assertTrue(_is_control("""\u0005"""			)			)

		self.assertFalse(_is_control("""A"""			)			)
		self.assertFalse(_is_control(""" """			)			)
		self.assertFalse(_is_control("""\t"""			)			)
		self.assertFalse(_is_control("""\r"""			)			)







	def  _a   (  self      :							Optional[Any]			)		-> Optional[Any]:
		"""simple docstring"""







		self.assertTrue(_is_punctuation("""-"""			)			)
		self.assertTrue(_is_punctuation("""$"""			)			)
		self.assertTrue(_is_punctuation("""`"""			)			)
		self.assertTrue(_is_punctuation("""."""			)			)

		self.assertFalse(_is_punctuation("""A"""			)			)
		self.assertFalse(_is_punctuation(""" """			)			)







	@slow
	def  _a   (  self      :							str			)		-> Optional[int]:
		"""simple docstring"""







		__lowercase             =  self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""			)

		__lowercase             =  tokenizer.encode("""sequence builders"""       ,				add_special_tokens=_lowerCAmelCase			)
		__lowercase             =  tokenizer.encode("""multi-sequence build"""       ,				add_special_tokens=_lowerCAmelCase			)

		__lowercase             =  tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase			)
		__lowercase             =  tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase       ,				_lowerCAmelCase			)

		assert encoded_sentence == text + [102]
		assert encoded_pair == text + [102] + text_a + [102]









def 				snake_case  (     lowerCamelCase					):




	'''simple docstring'''







	__lowercase             =  """"""
	for ch in key:
		if ch == " " or ch not in key_no_dups and ch.isalpha():
			key_no_dups += ch
	return key_no_dups





def 				snake_case  (     lowerCamelCase					):




	'''simple docstring'''







	__lowercase             =  [chr(i + 65					) for i in range(26					)]
	# Remove duplicate characters from key
	__lowercase             =  remove_duplicates(key.upper()					)
	__lowercase             =  len(lowerCamelCase					)
	# First fill cipher with key characters
	__lowercase             =  {alphabet[i]: char for i, char in enumerate(lowerCamelCase					)}
	# Then map remaining characters in alphabet to
	# the alphabet from the beginning
	for i in range(len(lowerCamelCase					)				,  26					):
		__lowercase             =  alphabet[i - offset]
		# Ensure we are not mapping letters to letters previously mapped
		while char in key:
			offset -= 1
			__lowercase             =  alphabet[i - offset]
		__lowercase             =  char
	return cipher_alphabet





def 				snake_case  (     lowerCamelCase				,  lowerCamelCase					):




	'''simple docstring'''







	return "".join(cipher_map.get(lowerCamelCase				,  lowerCamelCase					) for ch in message.upper()					)





def 				snake_case  (     lowerCamelCase				,  lowerCamelCase					):




	'''simple docstring'''







	__lowercase             =  {v: k for k, v in cipher_map.items()}
	return "".join(rev_cipher_map.get(lowerCamelCase				,  lowerCamelCase					) for ch in message.upper()					)





def 				snake_case  (     ):




	'''simple docstring'''







	__lowercase             =  input("""Enter message to encode or decode: """					).strip()
	__lowercase             =  input("""Enter keyword: """					).strip()
	__lowercase             =  input("""Encipher or decipher? E/D:"""					).strip()[0].lower()
	try:
		__lowercase             =  {"""e""": encipher, """d""": decipher}[option]
	except KeyError:
		raise KeyError("""invalid input option"""					)
	__lowercase             =  create_cipher_map(lowerCamelCase					)
	print(func(lowerCamelCase				,  lowerCamelCase					)					)


if __name__ == "__main__":
						import doctest

						doctest.testmod()
						main()

