import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 	__snake_case	( SCREAMING_SNAKE_CASE_ ,  unittest.TestCase		):





      '''simple docstring'''






      _snake_case    		=       BertJapaneseTokenizer
      _snake_case    		=       False
      _snake_case    		=       True



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any)				->Tuple:


          """simple docstring"""


          super().setUp()

          _lowerCamelCase     :		Dict 						=						[
              '''[UNK]''',
              '''[CLS]''',
              '''[SEP]''',
              '''こんにちは''',
              '''こん''',
              '''にちは''',
              '''ばんは''',
              '''##こん''',
              '''##にちは''',
              '''##ばんは''',
              '''世界''',
              '''##世界''',
              '''、''',
              '''##、''',
              '''。''',
              '''##。''',
          ]

          _lowerCamelCase     :		List[str] 						=						os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES["""vocab_file"""])
          with open(self.vocab_file    ,  """w"""    ,  encoding="""utf-8""") as vocab_writer:
              vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Tuple    ,  _UpperCamelCase     :				Union[str, Any])				->List[str]:


          """simple docstring"""


          _lowerCamelCase     :		Any 						=						'''こんにちは、世界。 \nこんばんは、世界。'''
          _lowerCamelCase     :		Dict 						=						'''こんにちは 、 世界 。 こんばんは 、 世界 。'''
          return input_text, output_text



      def 	_SCREAMING_SNAKE_CASE    (					self     :				str    ,  _UpperCamelCase     :				Any)				->Tuple:


          """simple docstring"""


          _lowerCamelCase     :		Dict 						=						self.get_input_output_texts(UpperCamelCase__)
          _lowerCamelCase     :		List[str] 						=						tokenizer.encode(UpperCamelCase__    ,  add_special_tokens=UpperCamelCase__)
          _lowerCamelCase     :		Dict 						=						tokenizer.decode(UpperCamelCase__    ,  clean_up_tokenization_spaces=UpperCamelCase__)
          return text, ids



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Dict)				->List[str]:


          """simple docstring"""


          pass  # TODO add if relevant



      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[Any])				->List[Any]:


          """simple docstring"""


          pass  # TODO add if relevant



      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[str])				->List[str]:


          """simple docstring"""


          pass  # TODO add if relevant



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any)				->Any:


          """simple docstring"""


          _lowerCamelCase     :		int 						=						self.tokenizer_class(self.vocab_file)

          _lowerCamelCase     :		Any 						=						tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。""")
          self.assertListEqual(UpperCamelCase__    ,  ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)    ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[Any])				->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase     :		Any 						=						self.tokenizer_class(self.vocab_file    ,  word_tokenizer_type="""mecab""")
          self.assertIsNotNone(UpperCamelCase__)

          _lowerCamelCase     :		Dict 						=						'''こんにちは、世界。\nこんばんは、世界。'''
          _lowerCamelCase     :		Dict 						=						tokenizer.tokenize(UpperCamelCase__)
          self.assertListEqual(UpperCamelCase__    ,  ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)    ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])

          _lowerCamelCase     :		Optional[int] 						=						os.path.join(self.tmpdirname    ,  """tokenizer.bin""")
          with open(UpperCamelCase__    ,  """wb""") as handle:
              pickle.dump(UpperCamelCase__    ,  UpperCamelCase__)

          with open(UpperCamelCase__    ,  """rb""") as handle:
              _lowerCamelCase     :		Tuple 						=						pickle.load(UpperCamelCase__)

          _lowerCamelCase     :		str 						=						tokenizer_new.tokenize(UpperCamelCase__)

          self.assertListEqual(UpperCamelCase__    ,  UpperCamelCase__)



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any])				->Optional[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Dict 						=						MecabTokenizer(mecab_dic="""ipadic""")

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,  )



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any])				->Optional[int]:


          """simple docstring"""


          try:
              _lowerCamelCase     :		Tuple 						=						MecabTokenizer(mecab_dic="""unidic_lite""")
          except ModuleNotFoundError:
              return

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,  )



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Tuple)				->Optional[int]:


          """simple docstring"""


          try:
              _lowerCamelCase     :		Optional[int] 						=						MecabTokenizer(mecab_dic="""unidic""")
          except ModuleNotFoundError:
              return

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,  )



      def 	_SCREAMING_SNAKE_CASE    (					self     :				int)				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						MecabTokenizer(do_lower_case=UpperCamelCase__    ,  mecab_dic="""ipadic""")

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,  )



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any])				->Optional[Any]:


          """simple docstring"""


          try:
              _lowerCamelCase     :		Dict 						=						MecabTokenizer(
                  do_lower_case=UpperCamelCase__    ,  normalize_text=UpperCamelCase__    ,  mecab_option="""-d /usr/local/lib/mecab/dic/jumandic""")
          except RuntimeError:
              # if dict doesn't exist in the system, previous code raises this error.
              return

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,  )



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Dict)				->Any:


          """simple docstring"""


          _lowerCamelCase     :		Optional[Any] 						=						MecabTokenizer(normalize_text=UpperCamelCase__    ,  mecab_dic="""ipadic""")

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]    ,  )



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any])				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Any 						=						self.tokenizer_class(self.vocab_file    ,  word_tokenizer_type="""sudachi""")
          self.assertIsNotNone(UpperCamelCase__)

          _lowerCamelCase     :		int 						=						'''こんにちは、世界。\nこんばんは、世界。'''
          _lowerCamelCase     :		Any 						=						tokenizer.tokenize(UpperCamelCase__)
          self.assertListEqual(UpperCamelCase__    ,  ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)    ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])

          _lowerCamelCase     :		str 						=						os.path.join(self.tmpdirname    ,  """tokenizer.bin""")
          with open(UpperCamelCase__    ,  """wb""") as handle:
              pickle.dump(UpperCamelCase__    ,  UpperCamelCase__)

          with open(UpperCamelCase__    ,  """rb""") as handle:
              _lowerCamelCase     :		Optional[int] 						=						pickle.load(UpperCamelCase__)

          _lowerCamelCase     :		Any 						=						tokenizer_new.tokenize(UpperCamelCase__)

          self.assertListEqual(UpperCamelCase__    ,  UpperCamelCase__)



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[Any])				->Tuple:


          """simple docstring"""


          _lowerCamelCase     :		str 						=						SudachiTokenizer(sudachi_dict_type="""core""")

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  [""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,  )



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Dict)				->Optional[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Optional[Any] 						=						SudachiTokenizer(sudachi_dict_type="""core"""    ,  sudachi_split_mode="""A""")

          self.assertListEqual(tokenizer.tokenize("""外国人参政権""")    ,  ["""外国""", """人""", """参政""", """権"""])



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Tuple)				->Dict:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						SudachiTokenizer(sudachi_dict_type="""core"""    ,  sudachi_split_mode="""B""")

          self.assertListEqual(tokenizer.tokenize("""外国人参政権""")    ,  ["""外国人""", """参政権"""])



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Dict)				->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase     :		str 						=						SudachiTokenizer(sudachi_dict_type="""core"""    ,  sudachi_split_mode="""C""")

          self.assertListEqual(tokenizer.tokenize("""外国人参政権""")    ,  ["""外国人参政権"""])



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any)				->int:


          """simple docstring"""


          _lowerCamelCase     :		Optional[int] 						=						SudachiTokenizer(do_lower_case=UpperCamelCase__    ,  sudachi_dict_type="""core""")

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  [""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,  )



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[int])				->int:


          """simple docstring"""


          _lowerCamelCase     :		Optional[int] 						=						SudachiTokenizer(normalize_text=UpperCamelCase__    ,  sudachi_dict_type="""core""")

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  [""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]    ,  )



      @require_sudachi
      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[str])				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Optional[int] 						=						SudachiTokenizer(trim_whitespace=UpperCamelCase__    ,  sudachi_dict_type="""core""")

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,  )



      @require_jumanpp
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any)				->str:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						self.tokenizer_class(self.vocab_file    ,  word_tokenizer_type="""jumanpp""")
          self.assertIsNotNone(UpperCamelCase__)

          _lowerCamelCase     :		Dict 						=						'''こんにちは、世界。\nこんばんは、世界。'''
          _lowerCamelCase     :		List[str] 						=						tokenizer.tokenize(UpperCamelCase__)
          self.assertListEqual(UpperCamelCase__    ,  ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)    ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])

          _lowerCamelCase     :		Dict 						=						os.path.join(self.tmpdirname    ,  """tokenizer.bin""")
          with open(UpperCamelCase__    ,  """wb""") as handle:
              pickle.dump(UpperCamelCase__    ,  UpperCamelCase__)

          with open(UpperCamelCase__    ,  """rb""") as handle:
              _lowerCamelCase     :		Dict 						=						pickle.load(UpperCamelCase__)

          _lowerCamelCase     :		Any 						=						tokenizer_new.tokenize(UpperCamelCase__)

          self.assertListEqual(UpperCamelCase__    ,  UpperCamelCase__)



      @require_jumanpp
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any)				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Any 						=						JumanppTokenizer()

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,  )



      @require_jumanpp
      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[Any])				->Tuple:


          """simple docstring"""


          _lowerCamelCase     :		Union[str, Any] 						=						JumanppTokenizer(do_lower_case=UpperCamelCase__)

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,  )



      @require_jumanpp
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any)				->Any:


          """simple docstring"""


          _lowerCamelCase     :		int 						=						JumanppTokenizer(normalize_text=UpperCamelCase__)

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,  )



      @require_jumanpp
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[int])				->Optional[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Dict 						=						JumanppTokenizer(trim_whitespace=UpperCamelCase__)

          self.assertListEqual(
              tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)    ,  ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]    ,  )



      @require_jumanpp
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[Any])				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		str 						=						JumanppTokenizer()

          self.assertListEqual(
              tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。""")    ,  ["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]    ,  )



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any])				->int:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

          _lowerCamelCase     :		str 						=						{}
          for i, token in enumerate(UpperCamelCase__):
              _lowerCamelCase     :		str 						=						i
          _lowerCamelCase     :		Union[str, Any] 						=						WordpieceTokenizer(vocab=UpperCamelCase__    ,  unk_token="""[UNK]""")

          self.assertListEqual(tokenizer.tokenize("""""")    ,  [])

          self.assertListEqual(tokenizer.tokenize("""こんにちは""")    ,  ["""こんにちは"""])

          self.assertListEqual(tokenizer.tokenize("""こんばんは""")    ,  ["""こん""", """##ばんは"""])

          self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは""")    ,  ["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""])



      def 	_SCREAMING_SNAKE_CASE    (					self     :				str)				->int:


          """simple docstring"""


          _lowerCamelCase     :		Optional[Any] 						=						BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp""")
          _lowerCamelCase     :		str 						=						tokenizer.subword_tokenizer

          _lowerCamelCase     :		Tuple 						=						subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。""")
          self.assertListEqual(UpperCamelCase__    ,  ["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""])

          _lowerCamelCase     :		Tuple 						=						subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは""")
          self.assertListEqual(UpperCamelCase__    ,  ["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""])




      def 	_SCREAMING_SNAKE_CASE    (					self     :				int)				->str:


          """simple docstring"""


          _lowerCamelCase     :		Dict 						=						self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese""")

          _lowerCamelCase     :		Optional[int] 						=						tokenizer.encode("""ありがとう。"""    ,  add_special_tokens=UpperCamelCase__)
          _lowerCamelCase     :		Any 						=						tokenizer.encode("""どういたしまして。"""    ,  add_special_tokens=UpperCamelCase__)

          _lowerCamelCase     :		List[str] 						=						tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
          _lowerCamelCase     :		List[str] 						=						tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    ,  UpperCamelCase__)

          # 2 is for "[CLS]", 3 is for "[SEP]"
          assert encoded_sentence == [2] + text + [3]
          assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 	__snake_case	( SCREAMING_SNAKE_CASE_ ,  unittest.TestCase		):





      '''simple docstring'''






      _snake_case    		=       BertJapaneseTokenizer
      _snake_case    		=       False



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any])				->Union[str, Any]:


          """simple docstring"""


          super().setUp()

          _lowerCamelCase     :		Optional[Any] 						=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

          _lowerCamelCase     :		Optional[Any] 						=						os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES["""vocab_file"""])
          with open(self.vocab_file    ,  """w"""    ,  encoding="""utf-8""") as vocab_writer:
              vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))



      def 	_SCREAMING_SNAKE_CASE    (					self     :				str    ,  **_UpperCamelCase     :				str)				->Dict:


          """simple docstring"""


          return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,  subword_tokenizer_type="""character"""    ,  **UpperCamelCase__)



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Dict    ,  _UpperCamelCase     :				List[str])				->Tuple:


          """simple docstring"""


          _lowerCamelCase     :		int 						=						'''こんにちは、世界。 \nこんばんは、世界。'''
          _lowerCamelCase     :		Dict 						=						'''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
          return input_text, output_text



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[Any])				->Any:


          """simple docstring"""


          pass  # TODO add if relevant



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any)				->int:


          """simple docstring"""


          pass  # TODO add if relevant



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Dict)				->Dict:


          """simple docstring"""


          pass  # TODO add if relevant



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Tuple)				->str:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						self.tokenizer_class(self.vocab_file    ,  subword_tokenizer_type="""character""")

          _lowerCamelCase     :		str 						=						tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。""")
          self.assertListEqual(
              UpperCamelCase__    ,  ["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""])
          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(UpperCamelCase__)    ,  [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])



      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[Any])				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		List[str] 						=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

          _lowerCamelCase     :		Optional[int] 						=						{}
          for i, token in enumerate(UpperCamelCase__):
              _lowerCamelCase     :		Optional[Any] 						=						i
          _lowerCamelCase     :		Dict 						=						CharacterTokenizer(vocab=UpperCamelCase__    ,  unk_token="""[UNK]""")

          self.assertListEqual(tokenizer.tokenize("""""")    ,  [])

          self.assertListEqual(tokenizer.tokenize("""こんにちは""")    ,  ["""こ""", """ん""", """に""", """ち""", """は"""])

          self.assertListEqual(tokenizer.tokenize("""こんにちほ""")    ,  ["""こ""", """ん""", """に""", """ち""", """[UNK]"""])




      def 	_SCREAMING_SNAKE_CASE    (					self     :				str)				->Dict:


          """simple docstring"""


          _lowerCamelCase     :		List[Any] 						=						self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char""")

          _lowerCamelCase     :		Any 						=						tokenizer.encode("""ありがとう。"""    ,  add_special_tokens=UpperCamelCase__)
          _lowerCamelCase     :		str 						=						tokenizer.encode("""どういたしまして。"""    ,  add_special_tokens=UpperCamelCase__)

          _lowerCamelCase     :		List[Any] 						=						tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
          _lowerCamelCase     :		int 						=						tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    ,  UpperCamelCase__)

          # 2 is for "[CLS]", 3 is for "[SEP]"
          assert encoded_sentence == [2] + text + [3]
          assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 	__snake_case	( unittest.TestCase		):





      '''simple docstring'''






      def 	_SCREAMING_SNAKE_CASE    (					self     :				int)				->int:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						'''cl-tohoku/bert-base-japanese'''
          _lowerCamelCase     :		str 						=						AutoTokenizer.from_pretrained(UpperCamelCase__)
          self.assertIsInstance(UpperCamelCase__    ,  UpperCamelCase__)





class 	__snake_case	( unittest.TestCase		):





      '''simple docstring'''






      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[int])				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		str 						=						'''cl-tohoku/bert-base-japanese'''
          with self.assertLogs("""transformers"""    ,  level="""WARNING""") as cm:
              BertTokenizer.from_pretrained(UpperCamelCase__)
              self.assertTrue(
                  cm.records[0].message.startswith(
                      """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                      """ is called from."""))
          _lowerCamelCase     :		Any 						=						'''bert-base-cased'''
          with self.assertLogs("""transformers"""    ,  level="""WARNING""") as cm:
              BertJapaneseTokenizer.from_pretrained(UpperCamelCase__)
              self.assertTrue(
                  cm.records[0].message.startswith(
                      """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                      """ is called from."""))





from __future__ import annotations

from math import pi
from typing import Protocol

import matplotlib.pyplot as plt
import numpy as np




class 	__snake_case	( __lowerCAmelCase		):





      '''simple docstring'''






      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[Any]    ,  _UpperCamelCase     :				float)				->float:


          """simple docstring"""


          return 0.0







def 						A__  (					__A   , __A	):







    '''simple docstring'''
    _lowerCamelCase     :		int 						=						min([-20, np.min(fft_results[1 : samplerate // 2 - 1]	)]	)
    _lowerCamelCase     :		Tuple 						=						max([20, np.max(fft_results[1 : samplerate // 2 - 1]	)]	)
    return lowest, highest







def 						A__  (					__A   , __A	):







    '''simple docstring'''
    _lowerCamelCase     :		Tuple 						=						512
    _lowerCamelCase     :		Tuple 						=						[1] + [0] * (size - 1)
    _lowerCamelCase     :		Optional[Any] 						=						[filter_type.process(__A	) for item in inputs]

    _lowerCamelCase     :		Optional[Any] 						=						[0] * (samplerate - size)  # zero-padding
    outputs += filler
    _lowerCamelCase     :		Tuple 						=						np.abs(np.fft.fft(__A	)	)
    _lowerCamelCase     :		List[Any] 						=						20 * np.logaa(__A	)

    # Frequencies on log scale from 24 to nyquist frequency
    plt.xlim(24   , samplerate / 2 - 1	)
    plt.xlabel("""Frequency (Hz)"""	)
    plt.xscale("""log"""	)

    # Display within reasonable bounds
    _lowerCamelCase     :		Any 						=						get_bounds(__A   , __A	)
    plt.ylim(max([-80, bounds[0]]	)   , min([80, bounds[1]]	)	)
    plt.ylabel("""Gain (dB)"""	)

    plt.plot(__A	)
    plt.show()







def 						A__  (					__A   , __A	):







    '''simple docstring'''
    _lowerCamelCase     :		Tuple 						=						512
    _lowerCamelCase     :		Union[str, Any] 						=						[1] + [0] * (size - 1)
    _lowerCamelCase     :		int 						=						[filter_type.process(__A	) for item in inputs]

    _lowerCamelCase     :		Optional[Any] 						=						[0] * (samplerate - size)  # zero-padding
    outputs += filler
    _lowerCamelCase     :		Any 						=						np.angle(np.fft.fft(__A	)	)

    # Frequencies on log scale from 24 to nyquist frequency
    plt.xlim(24   , samplerate / 2 - 1	)
    plt.xlabel("""Frequency (Hz)"""	)
    plt.xscale("""log"""	)

    plt.ylim(-2 * pi   , 2 * pi	)
    plt.ylabel("""Phase shift (Radians)"""	)
    plt.plot(np.unwrap(__A   , -2 * pi	)	)
    plt.show()




