





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class      lowercase__    (							__A     ,					unittest.TestCase  ):
  __UpperCamelCase	     =  BertJapaneseTokenizer
  __UpperCamelCase	     =  False
  __UpperCamelCase	     =  True







  def 	UpperCAmelCase__		(	self						):
       super().setUp()

       lowerCAmelCase_					:       Union[str, Any]    =		[
           """[UNK]""",
           """[CLS]""",
           """[SEP]""",
           """こんにちは""",
           """こん""",
           """にちは""",
           """ばんは""",
           """##こん""",
           """##にちは""",
           """##ばんは""",
           """世界""",
           """##世界""",
           """、""",
           """##、""",
           """。""",
           """##。""",
       ]

       lowerCAmelCase_					:       List[str]    =		os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["""vocab_file"""]						)
       with open(self.vocab_file    ,						"""w"""    ,						encoding="""utf-8"""						) as vocab_writer:
            vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       Tuple    =		"""こんにちは、世界。 \nこんばんは、世界。"""
       lowerCAmelCase_					:       Optional[Any]    =		"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
       return input_text, output_text







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_   ,  lowerCAmelCase_					:       int    =		self.get_input_output_texts(_lowercase						)
       lowerCAmelCase_					:       Tuple    =		tokenizer.encode(_lowercase    ,						add_special_tokens=_lowercase						)
       lowerCAmelCase_					:       str    =		tokenizer.decode(_lowercase    ,						clean_up_tokenization_spaces=_lowercase						)
       return text, ids







  def 	UpperCAmelCase__		(	self						):
       pass  # TODO add if relevant







  def 	UpperCAmelCase__		(	self						):
       pass  # TODO add if relevant







  def 	UpperCAmelCase__		(	self						):
       pass  # TODO add if relevant







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		self.tokenizer_class(self.vocab_file						)

       lowerCAmelCase_					:       List[Any]    =		tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""						)
       self.assertListEqual(_lowercase    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase						)    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Dict    =		self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="""mecab"""						)
       self.assertIsNotNone(_lowercase						)

       lowerCAmelCase_					:       Any    =		"""こんにちは、世界。\nこんばんは、世界。"""
       lowerCAmelCase_					:       Optional[Any]    =		tokenizer.tokenize(_lowercase						)
       self.assertListEqual(_lowercase    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase						)    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

       lowerCAmelCase_					:       List[str]    =		os.path.join(self.tmpdirname    ,						"""tokenizer.bin"""						)
       with open(_lowercase    ,						"""wb"""						) as handle:
            pickle.dump(_lowercase    ,						_lowercase						)

       with open(_lowercase    ,						"""rb"""						) as handle:
            lowerCAmelCase_					:       List[Any]    =		pickle.load(_lowercase						)

       lowerCAmelCase_					:       Any    =		tokenizer_new.tokenize(_lowercase						)

       self.assertListEqual(_lowercase    ,						_lowercase						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[Any]    =		MecabTokenizer(mecab_dic="""ipadic"""						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)







  def 	UpperCAmelCase__		(	self						):
       try:
            lowerCAmelCase_					:       List[Any]    =		MecabTokenizer(mecab_dic="""unidic_lite"""						)
       except ModuleNotFoundError:
            return

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)







  def 	UpperCAmelCase__		(	self						):
       try:
            lowerCAmelCase_					:       str    =		MecabTokenizer(mecab_dic="""unidic"""						)
       except ModuleNotFoundError:
            return

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[str]    =		MecabTokenizer(do_lower_case=_lowercase    ,						mecab_dic="""ipadic"""						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)







  def 	UpperCAmelCase__		(	self						):
       try:
            lowerCAmelCase_					:       Optional[int]    =		MecabTokenizer(
                do_lower_case=_lowercase    ,						normalize_text=_lowercase    ,						mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""						)
       except RuntimeError:
            # if dict doesn't exist in the system, previous code raises this error.
            return

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[Any]    =		MecabTokenizer(normalize_text=_lowercase    ,						mecab_dic="""ipadic"""						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]    ,						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[str]    =		self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="""sudachi"""						)
       self.assertIsNotNone(_lowercase						)

       lowerCAmelCase_					:       Optional[int]    =		"""こんにちは、世界。\nこんばんは、世界。"""
       lowerCAmelCase_					:       int    =		tokenizer.tokenize(_lowercase						)
       self.assertListEqual(_lowercase    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase						)    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

       lowerCAmelCase_					:       Dict    =		os.path.join(self.tmpdirname    ,						"""tokenizer.bin"""						)
       with open(_lowercase    ,						"""wb"""						) as handle:
            pickle.dump(_lowercase    ,						_lowercase						)

       with open(_lowercase    ,						"""rb"""						) as handle:
            lowerCAmelCase_					:       List[str]    =		pickle.load(_lowercase						)

       lowerCAmelCase_					:       Any    =		tokenizer_new.tokenize(_lowercase						)

       self.assertListEqual(_lowercase    ,						_lowercase						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		SudachiTokenizer(sudachi_dict_type="""core"""						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		SudachiTokenizer(sudachi_dict_type="""core"""    ,						sudachi_split_mode="""A"""						)

       self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)    ,						["""外国""", """人""", """参政""", """権"""]						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Union[str, Any]    =		SudachiTokenizer(sudachi_dict_type="""core"""    ,						sudachi_split_mode="""B"""						)

       self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)    ,						["""外国人""", """参政権"""]						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		SudachiTokenizer(sudachi_dict_type="""core"""    ,						sudachi_split_mode="""C"""						)

       self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)    ,						["""外国人参政権"""]						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[Any]    =		SudachiTokenizer(do_lower_case=_lowercase    ,						sudachi_dict_type="""core"""						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Dict    =		SudachiTokenizer(normalize_text=_lowercase    ,						sudachi_dict_type="""core"""						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]    ,						)







  @require_sudachi
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[str]    =		SudachiTokenizer(trim_whitespace=_lowercase    ,						sudachi_dict_type="""core"""						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)







  @require_jumanpp
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[str]    =		self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="""jumanpp"""						)
       self.assertIsNotNone(_lowercase						)

       lowerCAmelCase_					:       Optional[Any]    =		"""こんにちは、世界。\nこんばんは、世界。"""
       lowerCAmelCase_					:       Tuple    =		tokenizer.tokenize(_lowercase						)
       self.assertListEqual(_lowercase    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase						)    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

       lowerCAmelCase_					:       Dict    =		os.path.join(self.tmpdirname    ,						"""tokenizer.bin"""						)
       with open(_lowercase    ,						"""wb"""						) as handle:
            pickle.dump(_lowercase    ,						_lowercase						)

       with open(_lowercase    ,						"""rb"""						) as handle:
            lowerCAmelCase_					:       Optional[Any]    =		pickle.load(_lowercase						)

       lowerCAmelCase_					:       Optional[int]    =		tokenizer_new.tokenize(_lowercase						)

       self.assertListEqual(_lowercase    ,						_lowercase						)







  @require_jumanpp
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)







  @require_jumanpp
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[Any]    =		JumanppTokenizer(do_lower_case=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)







  @require_jumanpp
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[str]    =		JumanppTokenizer(normalize_text=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)







  @require_jumanpp
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		JumanppTokenizer(trim_whitespace=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]    ,						)







  @require_jumanpp
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[str]    =		JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""						)    ,						["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]    ,						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Tuple    =		["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

       lowerCAmelCase_					:       Tuple    =		{}
       for i, token in enumerate(_lowercase						):
            lowerCAmelCase_					:       Any    =		i
       lowerCAmelCase_					:       Tuple    =		WordpieceTokenizer(vocab=_lowercase    ,						unk_token="""[UNK]"""						)

       self.assertListEqual(tokenizer.tokenize(""""""						)    ,						[]						)

       self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)    ,						["""こんにちは"""]						)

       self.assertListEqual(tokenizer.tokenize("""こんばんは"""						)    ,						["""こん""", """##ばんは"""]						)

       self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""						)    ,						["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[str]    =		BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""						)
       lowerCAmelCase_					:       Optional[Any]    =		tokenizer.subword_tokenizer

       lowerCAmelCase_					:       Union[str, Any]    =		subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""						)
       self.assertListEqual(_lowercase    ,						["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]						)

       lowerCAmelCase_					:       Optional[Any]    =		subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""						)
       self.assertListEqual(_lowercase    ,						["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Dict    =		self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""						)

       lowerCAmelCase_					:       Union[str, Any]    =		tokenizer.encode("""ありがとう。"""    ,						add_special_tokens=_lowercase						)
       lowerCAmelCase_					:       str    =		tokenizer.encode("""どういたしまして。"""    ,						add_special_tokens=_lowercase						)

       lowerCAmelCase_					:       Any    =		tokenizer.build_inputs_with_special_tokens(_lowercase						)
       lowerCAmelCase_					:       Optional[int]    =		tokenizer.build_inputs_with_special_tokens(_lowercase    ,						_lowercase						)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class      lowercase__    (							__A     ,					unittest.TestCase  ):
  __UpperCamelCase	     =  BertJapaneseTokenizer
  __UpperCamelCase	     =  False







  def 	UpperCAmelCase__		(	self						):
       super().setUp()

       lowerCAmelCase_					:       int    =		["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

       lowerCAmelCase_					:       Optional[Any]    =		os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["""vocab_file"""]						)
       with open(self.vocab_file    ,						"""w"""    ,						encoding="""utf-8"""						) as vocab_writer:
            vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)







  def 	UpperCAmelCase__		(	self    ,						**_lowercase						):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						subword_tokenizer_type="""character"""    ,						**_lowercase						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       str    =		"""こんにちは、世界。 \nこんばんは、世界。"""
       lowerCAmelCase_					:       str    =		"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
       return input_text, output_text







  def 	UpperCAmelCase__		(	self						):
       pass  # TODO add if relevant







  def 	UpperCAmelCase__		(	self						):
       pass  # TODO add if relevant







  def 	UpperCAmelCase__		(	self						):
       pass  # TODO add if relevant







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Union[str, Any]    =		self.tokenizer_class(self.vocab_file    ,						subword_tokenizer_type="""character"""						)

       lowerCAmelCase_					:       Any    =		tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""						)
       self.assertListEqual(
           _lowercase    ,						["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]						)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_lowercase						)    ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Dict    =		["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

       lowerCAmelCase_					:       Optional[Any]    =		{}
       for i, token in enumerate(_lowercase						):
            lowerCAmelCase_					:       List[Any]    =		i
       lowerCAmelCase_					:       Any    =		CharacterTokenizer(vocab=_lowercase    ,						unk_token="""[UNK]"""						)

       self.assertListEqual(tokenizer.tokenize(""""""						)    ,						[]						)

       self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)    ,						["""こ""", """ん""", """に""", """ち""", """は"""]						)

       self.assertListEqual(tokenizer.tokenize("""こんにちほ"""						)    ,						["""こ""", """ん""", """に""", """ち""", """[UNK]"""]						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""						)

       lowerCAmelCase_					:       Any    =		tokenizer.encode("""ありがとう。"""    ,						add_special_tokens=_lowercase						)
       lowerCAmelCase_					:       Any    =		tokenizer.encode("""どういたしまして。"""    ,						add_special_tokens=_lowercase						)

       lowerCAmelCase_					:       str    =		tokenizer.build_inputs_with_special_tokens(_lowercase						)
       lowerCAmelCase_					:       List[str]    =		tokenizer.build_inputs_with_special_tokens(_lowercase    ,						_lowercase						)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class      lowercase__    (							unittest.TestCase  ):







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[Any]    =		"""cl-tohoku/bert-base-japanese"""
       lowerCAmelCase_					:       List[Any]    =		AutoTokenizer.from_pretrained(_lowercase						)
       self.assertIsInstance(_lowercase    ,						_lowercase						)





class      lowercase__    (							unittest.TestCase  ):







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		"""cl-tohoku/bert-base-japanese"""
       with self.assertLogs("""transformers"""    ,						level="""WARNING"""						) as cm:
            BertTokenizer.from_pretrained(_lowercase						)
            self.assertTrue(
                cm.records[0].message.startswith(
                    """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                    """ is called from."""						)						)
       lowerCAmelCase_					:       str    =		"""bert-base-cased"""
       with self.assertLogs("""transformers"""    ,						level="""WARNING"""						) as cm:
            BertJapaneseTokenizer.from_pretrained(_lowercase						)
            self.assertTrue(
                cm.records[0].message.startswith(
                    """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                    """ is called from."""						)						)







from dataclasses import dataclass
from typing import Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, randn_tensor
from .scheduling_utils import SchedulerMixin





@dataclass
class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  42
  __UpperCamelCase	     =  42
  __UpperCamelCase	     =  None





class      lowercase__    (							__A     ,					__A  ):
  __UpperCamelCase	     =  2







  @register_to_config
  def __init__(	self    ,						_lowercase = 0.02    ,						_lowercase = 100    ,						_lowercase = 1.007    ,						_lowercase = 80    ,						_lowercase = 0.05    ,						_lowercase = 50    ,						):
       # standard deviation of the initial noise distribution
       lowerCAmelCase_					:       int    =		sigma_max

       # setable values
       lowerCAmelCase_					:       int    =		None
       lowerCAmelCase_					:       np.IntTensor    =		None
       lowerCAmelCase_					:       torch.FloatTensor    =		None  # sigma(t_i)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = None						):
       return sample







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = None						):
       lowerCAmelCase_					:       int    =		num_inference_steps
       lowerCAmelCase_					:       int    =		np.arange(0    ,						self.num_inference_steps						)[::-1].copy()
       lowerCAmelCase_					:       List[Any]    =		torch.from_numpy(_lowercase						).to(_lowercase						)
       lowerCAmelCase_					:       List[str]    =		[
           (
               self.config.sigma_max**2
               * (self.config.sigma_min**2 / self.config.sigma_max**2) ** (i / (num_inference_steps - 1))
           )
           for i in self.timesteps
       ]
       lowerCAmelCase_					:       Optional[int]    =		torch.tensor(_lowercase    ,						dtype=torch.floataa    ,						device=_lowercase						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase = None						):
       if self.config.s_min <= sigma <= self.config.s_max:
            lowerCAmelCase_					:       List[Any]    =		min(self.config.s_churn / self.num_inference_steps    ,						2**0.5 - 1						)
       else:
            lowerCAmelCase_					:       List[str]    =		0

       # sample eps ~ N(0, S_noise^2 * I)
       lowerCAmelCase_					:       List[Any]    =		self.config.s_noise * randn_tensor(sample.shape    ,						generator=_lowercase						).to(sample.device						)
       lowerCAmelCase_					:       Optional[int]    =		sigma + gamma * sigma
       lowerCAmelCase_					:       int    =		sample + ((sigma_hat**2 - sigma**2) ** 0.5 * eps)

       return sample_hat, sigma_hat







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase = True    ,						):
       lowerCAmelCase_					:       Union[str, Any]    =		sample_hat + sigma_hat * model_output
       lowerCAmelCase_					:       Dict    =		(sample_hat - pred_original_sample) / sigma_hat
       lowerCAmelCase_					:       List[Any]    =		sample_hat + (sigma_prev - sigma_hat) * derivative

       if not return_dict:
            return (sample_prev, derivative)

       return KarrasVeOutput(
           prev_sample=_lowercase    ,						derivative=_lowercase    ,						pred_original_sample=_lowercase						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase = True    ,						):
       lowerCAmelCase_					:       str    =		sample_prev + sigma_prev * model_output
       lowerCAmelCase_					:       List[Any]    =		(sample_prev - pred_original_sample) / sigma_prev
       lowerCAmelCase_					:       Tuple    =		sample_hat + (sigma_prev - sigma_hat) * (0.5 * derivative + 0.5 * derivative_corr)

       if not return_dict:
            return (sample_prev, derivative)

       return KarrasVeOutput(
           prev_sample=_lowercase    ,						derivative=_lowercase    ,						pred_original_sample=_lowercase						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       raise NotImplementedError()
