






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase      :      Tuple     						=						get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
@require_tokenizers
class        __UpperCamelCase    (     _lowerCAmelCase   ,			unittest.TestCase   ):
 __snake_case	:Any   =       XLNetTokenizer
 __snake_case	:int   =       XLNetTokenizerFast
 __snake_case	:Union[str, Any]   =       True
 __snake_case	:str   =       True







 def  _a   (  self      :							Optional[Any]			)		-> Tuple:
  """simple docstring"""







  super().setUp()

  # We have a SentencePiece fixture for testing
  __lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				keep_accents=_lowerCAmelCase			)
  tokenizer.sanitize_special_tokens()
  tokenizer.save_pretrained(self.tmpdirname			)







 def  _a   (  self      :							Tuple			)		-> Optional[Any]:
  """simple docstring"""







  __lowercase             =  """<s>"""
  __lowercase             =  1

  self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase			)       ,				_lowerCAmelCase			)
  self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase			)       ,				_lowerCAmelCase			)







 def  _a   (  self      :							Tuple			)		-> Optional[int]:
  """simple docstring"""







  __lowercase             =  list(self.get_tokenizer().get_vocab().keys()			)

  self.assertEqual(vocab_keys[0]       ,				"""<unk>"""			)
  self.assertEqual(vocab_keys[1]       ,				"""<s>"""			)
  self.assertEqual(vocab_keys[-1]       ,				"""<eod>"""			)
  self.assertEqual(len(_lowerCAmelCase			)       ,				1006			)







 def  _a   (  self      :							Optional[int]			)		-> List[str]:
  """simple docstring"""







  self.assertEqual(self.get_tokenizer().vocab_size       ,				1000			)







 def  _a   (  self      :							Optional[int]			)		-> Optional[int]:
  """simple docstring"""







  __lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				keep_accents=_lowerCAmelCase			)

  __lowercase             =  tokenizer.tokenize("""This is a test"""			)
  self.assertListEqual(_lowerCAmelCase       ,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

  self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase			)       ,				[285, 46, 10, 170, 382]			)

  __lowercase             =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
  self.assertListEqual(
      _lowerCAmelCase       ,				[
          SPIECE_UNDERLINE + """I""",
          SPIECE_UNDERLINE + """was""",
          SPIECE_UNDERLINE + """b""",
          """or""",
          """n""",
          SPIECE_UNDERLINE + """in""",
          SPIECE_UNDERLINE + """""",
          """9""",
          """2""",
          """0""",
          """0""",
          """0""",
          """,""",
          SPIECE_UNDERLINE + """and""",
          SPIECE_UNDERLINE + """this""",
          SPIECE_UNDERLINE + """is""",
          SPIECE_UNDERLINE + """f""",
          """al""",
          """s""",
          """é""",
          """.""",
      ]       ,				)
  __lowercase             =  tokenizer.convert_tokens_to_ids(_lowerCAmelCase			)
  self.assertListEqual(_lowerCAmelCase       ,				[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			)

  __lowercase             =  tokenizer.convert_ids_to_tokens(_lowerCAmelCase			)
  self.assertListEqual(
      _lowerCAmelCase       ,				[
          SPIECE_UNDERLINE + """I""",
          SPIECE_UNDERLINE + """was""",
          SPIECE_UNDERLINE + """b""",
          """or""",
          """n""",
          SPIECE_UNDERLINE + """in""",
          SPIECE_UNDERLINE + """""",
          """<unk>""",
          """2""",
          """0""",
          """0""",
          """0""",
          """,""",
          SPIECE_UNDERLINE + """and""",
          SPIECE_UNDERLINE + """this""",
          SPIECE_UNDERLINE + """is""",
          SPIECE_UNDERLINE + """f""",
          """al""",
          """s""",
          """<unk>""",
          """.""",
      ]       ,				)







 def  _a   (  self      :							Optional[Any]			)		-> Tuple:
  """simple docstring"""







  __lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				do_lower_case=_lowerCAmelCase			)
  __lowercase             =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
  self.assertListEqual(
      _lowerCAmelCase       ,				[
          SPIECE_UNDERLINE + """""",
          """i""",
          SPIECE_UNDERLINE + """was""",
          SPIECE_UNDERLINE + """b""",
          """or""",
          """n""",
          SPIECE_UNDERLINE + """in""",
          SPIECE_UNDERLINE + """""",
          """9""",
          """2""",
          """0""",
          """0""",
          """0""",
          """,""",
          SPIECE_UNDERLINE + """and""",
          SPIECE_UNDERLINE + """this""",
          SPIECE_UNDERLINE + """is""",
          SPIECE_UNDERLINE + """f""",
          """al""",
          """se""",
          """.""",
      ]       ,				)
  self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)       ,				["""▁he""", """ll""", """o"""]			)







 def  _a   (  self      :							List[Any]			)		-> Tuple:
  """simple docstring"""







  __lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				do_lower_case=_lowerCAmelCase			)
  __lowercase             =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
  self.assertListEqual(
      _lowerCAmelCase       ,				[
          SPIECE_UNDERLINE + """I""",
          SPIECE_UNDERLINE + """was""",
          SPIECE_UNDERLINE + """b""",
          """or""",
          """n""",
          SPIECE_UNDERLINE + """in""",
          SPIECE_UNDERLINE + """""",
          """9""",
          """2""",
          """0""",
          """0""",
          """0""",
          """,""",
          SPIECE_UNDERLINE + """and""",
          SPIECE_UNDERLINE + """this""",
          SPIECE_UNDERLINE + """is""",
          SPIECE_UNDERLINE + """f""",
          """al""",
          """se""",
          """.""",
      ]       ,				)







 @slow
 def  _a   (  self      :							Union[str, Any]			)		-> Any:
  """simple docstring"""







  __lowercase             =  XLNetTokenizer.from_pretrained("""xlnet-base-cased"""			)

  __lowercase             =  tokenizer.encode("""sequence builders"""       ,				add_special_tokens=_lowerCAmelCase			)
  __lowercase             =  tokenizer.encode("""multi-sequence build"""       ,				add_special_tokens=_lowerCAmelCase			)

  __lowercase             =  tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase			)
  __lowercase             =  tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase       ,				_lowerCAmelCase			)

  assert encoded_sentence == text + [4, 3]
  assert encoded_pair == text + [4] + text_a + [4, 3]







 @slow
 def  _a   (  self      :							List[Any]			)		-> Union[str, Any]:
  """simple docstring"""







  __lowercase             =  {"""input_ids""": [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
  # fmt: on

  self.tokenizer_integration_test_util(
      expected_encoding=_lowerCAmelCase       ,				model_name="""xlnet-base-cased"""       ,				revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""       ,				)









def 				snake_case  (     lowerCamelCase					):




	'''simple docstring'''







	__lowercase             =  """"""
	for ch in key:
		if ch == " " or ch not in key_no_dups and ch.isalpha():
			key_no_dups += ch
	return key_no_dups





def 				snake_case  (     lowerCamelCase					):




	'''simple docstring'''







	__lowercase             =  [chr(i + 65					) for i in range(26					)]
	# Remove duplicate characters from key
	__lowercase             =  remove_duplicates(key.upper()					)
	__lowercase             =  len(lowerCamelCase					)
	# First fill cipher with key characters
	__lowercase             =  {alphabet[i]: char for i, char in enumerate(lowerCamelCase					)}
	# Then map remaining characters in alphabet to
	# the alphabet from the beginning
	for i in range(len(lowerCamelCase					)				,  26					):
		__lowercase             =  alphabet[i - offset]
		# Ensure we are not mapping letters to letters previously mapped
		while char in key:
			offset -= 1
			__lowercase             =  alphabet[i - offset]
		__lowercase             =  char
	return cipher_alphabet





def 				snake_case  (     lowerCamelCase				,  lowerCamelCase					):




	'''simple docstring'''







	return "".join(cipher_map.get(lowerCamelCase				,  lowerCamelCase					) for ch in message.upper()					)





def 				snake_case  (     lowerCamelCase				,  lowerCamelCase					):




	'''simple docstring'''







	__lowercase             =  {v: k for k, v in cipher_map.items()}
	return "".join(rev_cipher_map.get(lowerCamelCase				,  lowerCamelCase					) for ch in message.upper()					)





def 				snake_case  (     ):




	'''simple docstring'''







	__lowercase             =  input("""Enter message to encode or decode: """					).strip()
	__lowercase             =  input("""Enter keyword: """					).strip()
	__lowercase             =  input("""Encipher or decipher? E/D:"""					).strip()[0].lower()
	try:
		__lowercase             =  {"""e""": encipher, """d""": decipher}[option]
	except KeyError:
		raise KeyError("""invalid input option"""					)
	__lowercase             =  create_cipher_map(lowerCamelCase					)
	print(func(lowerCamelCase				,  lowerCamelCase					)					)


if __name__ == "__main__":
						import doctest

						doctest.testmod()
						main()

