






"""simple docstring"""



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase       ,		unittest.TestCase		):
							snake_case_     =							BertJapaneseTokenizer
							snake_case_     =							False
							snake_case_     =							True


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													super().setUp()

													A__						:  List[str]		  =  [
													    """[UNK]""",
													    """[CLS]""",
													    """[SEP]""",
													    """こんにちは""",
													    """こん""",
													    """にちは""",
													    """ばんは""",
													    """##こん""",
													    """##にちは""",
													    """##ばんは""",
													    """世界""",
													    """##世界""",
													    """、""",
													    """##、""",
													    """。""",
													    """##。""",
													]

													A__						:  List[str]		  =  os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES["""vocab_file"""]							)
													with open(self.vocab_file			,    """w"""			,    encoding="""utf-8"""							) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)


							def 						_UpperCamelCase       (			self      :						Union[str, Any]			,    snake_case      :						Any							):
													'''simple docstring'''





													A__						:  int		  =  """こんにちは、世界。 \nこんばんは、世界。"""
													A__						:  Optional[Any]		  =  """こんにちは 、 世界 。 こんばんは 、 世界 。"""
													return input_text, output_text


							def 						_UpperCamelCase       (			self      :						Optional[int]			,    snake_case      :						Tuple							):
													'''simple docstring'''





													A__  ,  A__						:  Optional[int]		  =  self.get_input_output_texts(snake_case							)
													A__						:  List[str]		  =  tokenizer.encode(snake_case			,    add_special_tokens=snake_case							)
													A__						:  List[Any]		  =  tokenizer.decode(snake_case			,    clean_up_tokenization_spaces=snake_case							)
													return text, ids


							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													pass  # TODO add if relevant


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													pass  # TODO add if relevant


							def 						_UpperCamelCase       (			self      :						Optional[int]							):
													'''simple docstring'''





													pass  # TODO add if relevant


							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  self.tokenizer_class(self.vocab_file							)

													A__						:  Dict		  =  tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""							)
													self.assertListEqual(snake_case			,    ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case							)			,    [3, 12, 10, 14, 4, 9, 12, 10, 14]							)


							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  self.tokenizer_class(self.vocab_file			,    word_tokenizer_type="""mecab"""							)
													self.assertIsNotNone(snake_case							)

													A__						:  Tuple		  =  """こんにちは、世界。\nこんばんは、世界。"""
													A__						:  Tuple		  =  tokenizer.tokenize(snake_case							)
													self.assertListEqual(snake_case			,    ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case							)			,    [3, 12, 10, 14, 4, 9, 12, 10, 14]							)

													A__						:  Any		  =  os.path.join(self.tmpdirname			,    """tokenizer.bin"""							)
													with open(snake_case			,    """wb"""							) as handle:
																			pickle.dump(snake_case			,    snake_case							)

													with open(snake_case			,    """rb"""							) as handle:
																			A__						:  int		  =  pickle.load(snake_case							)

													A__						:  List[str]		  =  tokenizer_new.tokenize(snake_case							)

													self.assertListEqual(snake_case			,    snake_case							)


							def 						_UpperCamelCase       (			self      :						Dict							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  MecabTokenizer(mecab_dic="""ipadic"""							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]			,    )


							def 						_UpperCamelCase       (			self      :						Optional[int]							):
													'''simple docstring'''





													try:
																			A__						:  Dict		  =  MecabTokenizer(mecab_dic="""unidic_lite"""							)
													except ModuleNotFoundError:
																			return

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]			,    )


							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													try:
																			A__						:  List[Any]		  =  MecabTokenizer(mecab_dic="""unidic"""							)
													except ModuleNotFoundError:
																			return

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]			,    )


							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													A__						:  str		  =  MecabTokenizer(do_lower_case=snake_case			,    mecab_dic="""ipadic"""							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]			,    )


							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													try:
																			A__						:  Any		  =  MecabTokenizer(
																			    do_lower_case=snake_case			,    normalize_text=snake_case			,    mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""							)
													except RuntimeError:
																			# if dict doesn't exist in the system, previous code raises this error.
																			return

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]			,    )


							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													A__						:  List[Any]		  =  MecabTokenizer(normalize_text=snake_case			,    mecab_dic="""ipadic"""							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]			,    )


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Dict		  =  self.tokenizer_class(self.vocab_file			,    word_tokenizer_type="""sudachi"""							)
													self.assertIsNotNone(snake_case							)

													A__						:  List[str]		  =  """こんにちは、世界。\nこんばんは、世界。"""
													A__						:  Optional[int]		  =  tokenizer.tokenize(snake_case							)
													self.assertListEqual(snake_case			,    ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case							)			,    [3, 12, 10, 14, 4, 9, 12, 10, 14]							)

													A__						:  Union[str, Any]		  =  os.path.join(self.tmpdirname			,    """tokenizer.bin"""							)
													with open(snake_case			,    """wb"""							) as handle:
																			pickle.dump(snake_case			,    snake_case							)

													with open(snake_case			,    """rb"""							) as handle:
																			A__						:  Tuple		  =  pickle.load(snake_case							)

													A__						:  Tuple		  =  tokenizer_new.tokenize(snake_case							)

													self.assertListEqual(snake_case			,    snake_case							)


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						Optional[int]							):
													'''simple docstring'''





													A__						:  Any		  =  SudachiTokenizer(sudachi_dict_type="""core"""							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    [""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]			,    )


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													A__						:  Dict		  =  SudachiTokenizer(sudachi_dict_type="""core"""			,    sudachi_split_mode="""A"""							)

													self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)			,    ["""外国""", """人""", """参政""", """権"""]							)


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  str		  =  SudachiTokenizer(sudachi_dict_type="""core"""			,    sudachi_split_mode="""B"""							)

													self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)			,    ["""外国人""", """参政権"""]							)


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						Dict							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  SudachiTokenizer(sudachi_dict_type="""core"""			,    sudachi_split_mode="""C"""							)

													self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)			,    ["""外国人参政権"""]							)


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						Optional[Any]							):
													'''simple docstring'''





													A__						:  Any		  =  SudachiTokenizer(do_lower_case=snake_case			,    sudachi_dict_type="""core"""							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    [""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]			,    )


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													A__						:  Tuple		  =  SudachiTokenizer(normalize_text=snake_case			,    sudachi_dict_type="""core"""							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    [""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]			,    )


							@require_sudachi
							def 						_UpperCamelCase       (			self      :						Optional[int]							):
													'''simple docstring'''





													A__						:  List[str]		  =  SudachiTokenizer(trim_whitespace=snake_case			,    sudachi_dict_type="""core"""							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]			,    )


							@require_jumanpp
							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  int		  =  self.tokenizer_class(self.vocab_file			,    word_tokenizer_type="""jumanpp"""							)
													self.assertIsNotNone(snake_case							)

													A__						:  Dict		  =  """こんにちは、世界。\nこんばんは、世界。"""
													A__						:  str		  =  tokenizer.tokenize(snake_case							)
													self.assertListEqual(snake_case			,    ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case							)			,    [3, 12, 10, 14, 4, 9, 12, 10, 14]							)

													A__						:  str		  =  os.path.join(self.tmpdirname			,    """tokenizer.bin"""							)
													with open(snake_case			,    """wb"""							) as handle:
																			pickle.dump(snake_case			,    snake_case							)

													with open(snake_case			,    """rb"""							) as handle:
																			A__						:  Union[str, Any]		  =  pickle.load(snake_case							)

													A__						:  List[Any]		  =  tokenizer_new.tokenize(snake_case							)

													self.assertListEqual(snake_case			,    snake_case							)


							@require_jumanpp
							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  JumanppTokenizer()

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]			,    )


							@require_jumanpp
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													A__						:  Dict		  =  JumanppTokenizer(do_lower_case=snake_case							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]			,    )


							@require_jumanpp
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													A__						:  int		  =  JumanppTokenizer(normalize_text=snake_case							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]			,    )


							@require_jumanpp
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													A__						:  str		  =  JumanppTokenizer(trim_whitespace=snake_case							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)			,    ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]			,    )


							@require_jumanpp
							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  JumanppTokenizer()

													self.assertListEqual(
													    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""							)			,    ["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]			,    )


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

													A__						:  Optional[Any]		  =  {}
													for i, token in enumerate(snake_case							):
																			A__						:  str		  =  i
													A__						:  Optional[Any]		  =  WordpieceTokenizer(vocab=snake_case			,    unk_token="""[UNK]"""							)

													self.assertListEqual(tokenizer.tokenize(""""""							)			,    []							)

													self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)			,    ["""こんにちは"""]							)

													self.assertListEqual(tokenizer.tokenize("""こんばんは"""							)			,    ["""こん""", """##ばんは"""]							)

													self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""							)			,    ["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]							)


							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													A__						:  Tuple		  =  BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""							)
													A__						:  Dict		  =  tokenizer.subword_tokenizer

													A__						:  Optional[int]		  =  subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""							)
													self.assertListEqual(snake_case			,    ["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]							)

													A__						:  Tuple		  =  subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""							)
													self.assertListEqual(snake_case			,    ["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]							)


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""							)

													A__						:  List[str]		  =  tokenizer.encode("""ありがとう。"""			,    add_special_tokens=snake_case							)
													A__						:  Union[str, Any]		  =  tokenizer.encode("""どういたしまして。"""			,    add_special_tokens=snake_case							)

													A__						:  int		  =  tokenizer.build_inputs_with_special_tokens(snake_case							)
													A__						:  Any		  =  tokenizer.build_inputs_with_special_tokens(snake_case			,    snake_case							)

													# 2 is for "[CLS]", 3 is for "[SEP]"
													assert encoded_sentence == [2] + text + [3]
													assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase       ,		unittest.TestCase		):
							snake_case_     =							BertJapaneseTokenizer
							snake_case_     =							False


							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													super().setUp()

													A__						:  Dict		  =  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

													A__						:  Tuple		  =  os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES["""vocab_file"""]							)
													with open(self.vocab_file			,    """w"""			,    encoding="""utf-8"""							) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)


							def 						_UpperCamelCase       (			self      :						List[str]			,    **snake_case      :						List[str]							):
													'''simple docstring'''





													return BertJapaneseTokenizer.from_pretrained(self.tmpdirname			,    subword_tokenizer_type="""character"""			,    **snake_case							)


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						List[Any]							):
													'''simple docstring'''





													A__						:  int		  =  """こんにちは、世界。 \nこんばんは、世界。"""
													A__						:  str		  =  """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
													return input_text, output_text


							def 						_UpperCamelCase       (			self      :						Optional[int]							):
													'''simple docstring'''





													pass  # TODO add if relevant


							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													pass  # TODO add if relevant


							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													pass  # TODO add if relevant


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  self.tokenizer_class(self.vocab_file			,    subword_tokenizer_type="""character"""							)

													A__						:  Tuple		  =  tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""							)
													self.assertListEqual(
													    snake_case			,    ["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]							)
													self.assertListEqual(
													    tokenizer.convert_tokens_to_ids(snake_case							)			,    [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]							)


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

													A__						:  Tuple		  =  {}
													for i, token in enumerate(snake_case							):
																			A__						:  Union[str, Any]		  =  i
													A__						:  Union[str, Any]		  =  CharacterTokenizer(vocab=snake_case			,    unk_token="""[UNK]"""							)

													self.assertListEqual(tokenizer.tokenize(""""""							)			,    []							)

													self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)			,    ["""こ""", """ん""", """に""", """ち""", """は"""]							)

													self.assertListEqual(tokenizer.tokenize("""こんにちほ"""							)			,    ["""こ""", """ん""", """に""", """ち""", """[UNK]"""]							)


							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													A__						:  str		  =  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""							)

													A__						:  Tuple		  =  tokenizer.encode("""ありがとう。"""			,    add_special_tokens=snake_case							)
													A__						:  Dict		  =  tokenizer.encode("""どういたしまして。"""			,    add_special_tokens=snake_case							)

													A__						:  Tuple		  =  tokenizer.build_inputs_with_special_tokens(snake_case							)
													A__						:  List[Any]		  =  tokenizer.build_inputs_with_special_tokens(snake_case			,    snake_case							)

													# 2 is for "[CLS]", 3 is for "[SEP]"
													assert encoded_sentence == [2] + text + [3]
													assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													A__						:  str		  =  """cl-tohoku/bert-base-japanese"""
													A__						:  Optional[Any]		  =  AutoTokenizer.from_pretrained(snake_case							)
													self.assertIsInstance(snake_case			,    snake_case							)






class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


							def 						_UpperCamelCase       (			self      :						Optional[Any]							):
													'''simple docstring'''





													A__						:  Any		  =  """cl-tohoku/bert-base-japanese"""
													with self.assertLogs("""transformers"""			,    level="""WARNING"""							) as cm:
																			BertTokenizer.from_pretrained(snake_case							)
																			self.assertTrue(
																			    cm.records[0].message.startswith(
																			        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
																			        """ is called from."""							)							)
													A__						:  Optional[int]		  =  """bert-base-cased"""
													with self.assertLogs("""transformers"""			,    level="""WARNING"""							) as cm:
																			BertJapaneseTokenizer.from_pretrained(snake_case							)
																			self.assertTrue(
																			    cm.records[0].message.startswith(
																			        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
																			        """ is called from."""							)							)














"""simple docstring"""



import unittest

import numpy as np
import torch

from diffusers import ScoreSdeVePipeline, ScoreSdeVeScheduler, UNetaDModel
from diffusers.utils.testing_utils import enable_full_determinism, require_torch, slow, torch_device


enable_full_determinism()






class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


							@property
							def 						_UpperCamelCase       (			self      :						Dict							):
													'''simple docstring'''





													torch.manual_seed(0							)
													A__						:  Optional[Any]		  =  UNetaDModel(
													    block_out_channels=(32, 64)			,    layers_per_block=2			,    sample_size=32			,    in_channels=3			,    out_channels=3			,    down_block_types=("""DownBlock2D""", """AttnDownBlock2D""")			,    up_block_types=("""AttnUpBlock2D""", """UpBlock2D""")			,    )
													return model


							def 						_UpperCamelCase       (			self      :						Optional[Any]							):
													'''simple docstring'''





													A__						:  List[Any]		  =  self.dummy_uncond_unet
													A__						:  Dict		  =  ScoreSdeVeScheduler()

													A__						:  str		  =  ScoreSdeVePipeline(unet=snake_case			,    scheduler=snake_case							)
													sde_ve.to(snake_case							)
													sde_ve.set_progress_bar_config(disable=snake_case							)

													A__						:  Optional[int]		  =  torch.manual_seed(0							)
													A__						:  Any		  =  sde_ve(num_inference_steps=2			,    output_type="""numpy"""			,    generator=snake_case							).images

													A__						:  List[str]		  =  torch.manual_seed(0							)
													A__						:  int		  =  sde_ve(num_inference_steps=2			,    output_type="""numpy"""			,    generator=snake_case			,    return_dict=snake_case							)[
													    0
													]

													A__						:  int		  =  image[0, -3:, -3:, -1]
													A__						:  Optional[Any]		  =  image_from_tuple[0, -3:, -3:, -1]

													assert image.shape == (1, 32, 32, 3)
													A__						:  Optional[Any]		  =  np.array([0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]							)

													assert np.abs(image_slice.flatten() - expected_slice							).max() < 1e-2
													assert np.abs(image_from_tuple_slice.flatten() - expected_slice							).max() < 1e-2






@slow
@require_torch
class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  """google/ncsnpp-church-256"""
													A__						:  Tuple		  =  UNetaDModel.from_pretrained(snake_case							)

													A__						:  int		  =  ScoreSdeVeScheduler.from_pretrained(snake_case							)

													A__						:  List[Any]		  =  ScoreSdeVePipeline(unet=snake_case			,    scheduler=snake_case							)
													sde_ve.to(snake_case							)
													sde_ve.set_progress_bar_config(disable=snake_case							)

													A__						:  Dict		  =  torch.manual_seed(0							)
													A__						:  Optional[int]		  =  sde_ve(num_inference_steps=10			,    output_type="""numpy"""			,    generator=snake_case							).images

													A__						:  Optional[int]		  =  image[0, -3:, -3:, -1]

													assert image.shape == (1, 256, 256, 3)

													A__						:  Dict		  =  np.array([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0]							)

													assert np.abs(image_slice.flatten() - expected_slice							).max() < 1e-2






