





'''simple docstring'''

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class       UpperCAmelCase (						_lowercase			,       unittest.TestCase  ):
		UpperCAmelCase					:	Any 			=		BertJapaneseTokenizer
		UpperCAmelCase					:	Any 			=		False
		UpperCAmelCase					:	Union[str, Any] 			=		True




		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> List[str]:
						super().setUp()

						lowercase         = [
						    "[UNK]",
						    "[CLS]",
						    "[SEP]",
						    "こんにちは",
						    "こん",
						    "にちは",
						    "ばんは",
						    "##こん",
						    "##にちは",
						    "##ばんは",
						    "世界",
						    "##世界",
						    "、",
						    "##、",
						    "。",
						    "##。",
						]

						lowercase         = os.path.join(self.tmpdirname							,			VOCAB_FILES_NAMES["vocab_file"]				)
						with open(self.vocab_file							,			"w"							,			encoding="utf-8"				) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)




		def 				UpperCAmelCase__   (self      :	int							,			A__      :	Optional[int]				)					-> Any:
						lowercase         = "こんにちは、世界。 \nこんばんは、世界。"
						lowercase         = "こんにちは 、 世界 。 こんばんは 、 世界 。"
						return input_text, output_text




		def 				UpperCAmelCase__   (self      :	int							,			A__      :	List[str]				)					-> Optional[Any]:
						lowercase   ,							lowercase         = self.get_input_output_texts(A__				)
						lowercase         = tokenizer.encode(A__							,			add_special_tokens=A__				)
						lowercase         = tokenizer.decode(A__							,			clean_up_tokenization_spaces=A__				)
						return text, ids




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Optional[Any]:
						pass  # TODO add if relevant




		def 				UpperCAmelCase__   (self      :	str				)					-> int:
						pass  # TODO add if relevant




		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> List[Any]:
						pass  # TODO add if relevant




		def 				UpperCAmelCase__   (self      :	Tuple				)					-> int:
						lowercase         = self.tokenizer_class(self.vocab_file				)

						lowercase         = tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"				)
						self.assertListEqual(A__							,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]				)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__				)							,			[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]				)




		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Optional[Any]:
						lowercase         = self.tokenizer_class(self.vocab_file							,			word_tokenizer_type="mecab"				)
						self.assertIsNotNone(A__				)

						lowercase         = "こんにちは、世界。\nこんばんは、世界。"
						lowercase         = tokenizer.tokenize(A__				)
						self.assertListEqual(A__							,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]				)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__				)							,			[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]				)

						lowercase         = os.path.join(self.tmpdirname							,			"tokenizer.bin"				)
						with open(A__							,			"wb"				) as handle:
										pickle.dump(A__							,			A__				)

						with open(A__							,			"rb"				) as handle:
										lowercase         = pickle.load(A__				)

						lowercase         = tokenizer_new.tokenize(A__				)

						self.assertListEqual(A__							,			A__				)




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Any:
						lowercase         = MecabTokenizer(mecab_dic="ipadic"				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,			)




		def 				UpperCAmelCase__   (self      :	Dict				)					-> str:
						try:
										lowercase         = MecabTokenizer(mecab_dic="unidic_lite"				)
						except ModuleNotFoundError:
										return

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,			)




		def 				UpperCAmelCase__   (self      :	Optional[int]				)					-> Dict:
						try:
										lowercase         = MecabTokenizer(mecab_dic="unidic"				)
						except ModuleNotFoundError:
										return

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,			)




		def 				UpperCAmelCase__   (self      :	str				)					-> str:
						lowercase         = MecabTokenizer(do_lower_case=A__							,			mecab_dic="ipadic"				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]							,			)




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Tuple:
						try:
										lowercase         = MecabTokenizer(
										    do_lower_case=A__							,			normalize_text=A__							,			mecab_option="-d /usr/local/lib/mecab/dic/jumandic"				)
						except RuntimeError:
										# if dict doesn't exist in the system, previous code raises this error.
										return

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]							,			)




		def 				UpperCAmelCase__   (self      :	Any				)					-> List[Any]:
						lowercase         = MecabTokenizer(normalize_text=A__							,			mecab_dic="ipadic"				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]							,			)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> str:
						lowercase         = self.tokenizer_class(self.vocab_file							,			word_tokenizer_type="sudachi"				)
						self.assertIsNotNone(A__				)

						lowercase         = "こんにちは、世界。\nこんばんは、世界。"
						lowercase         = tokenizer.tokenize(A__				)
						self.assertListEqual(A__							,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]				)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__				)							,			[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]				)

						lowercase         = os.path.join(self.tmpdirname							,			"tokenizer.bin"				)
						with open(A__							,			"wb"				) as handle:
										pickle.dump(A__							,			A__				)

						with open(A__							,			"rb"				) as handle:
										lowercase         = pickle.load(A__				)

						lowercase         = tokenizer_new.tokenize(A__				)

						self.assertListEqual(A__							,			A__				)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> List[str]:
						lowercase         = SudachiTokenizer(sudachi_dict_type="core"				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]							,			)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Optional[Any]:
						lowercase         = SudachiTokenizer(sudachi_dict_type="core"							,			sudachi_split_mode="A"				)

						self.assertListEqual(tokenizer.tokenize("外国人参政権"				)							,			["外国", "人", "参政", "権"]				)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> Any:
						lowercase         = SudachiTokenizer(sudachi_dict_type="core"							,			sudachi_split_mode="B"				)

						self.assertListEqual(tokenizer.tokenize("外国人参政権"				)							,			["外国人", "参政権"]				)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	Dict				)					-> Optional[int]:
						lowercase         = SudachiTokenizer(sudachi_dict_type="core"							,			sudachi_split_mode="C"				)

						self.assertListEqual(tokenizer.tokenize("外国人参政権"				)							,			["外国人参政権"]				)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Any:
						lowercase         = SudachiTokenizer(do_lower_case=A__							,			sudachi_dict_type="core"				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]							,			)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Optional[Any]:
						lowercase         = SudachiTokenizer(normalize_text=A__							,			sudachi_dict_type="core"				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]							,			)




		@require_sudachi
		def 				UpperCAmelCase__   (self      :	Any				)					-> str:
						lowercase         = SudachiTokenizer(trim_whitespace=A__							,			sudachi_dict_type="core"				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,			)




		@require_jumanpp
		def 				UpperCAmelCase__   (self      :	Optional[int]				)					-> Union[str, Any]:
						lowercase         = self.tokenizer_class(self.vocab_file							,			word_tokenizer_type="jumanpp"				)
						self.assertIsNotNone(A__				)

						lowercase         = "こんにちは、世界。\nこんばんは、世界。"
						lowercase         = tokenizer.tokenize(A__				)
						self.assertListEqual(A__							,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]				)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__				)							,			[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]				)

						lowercase         = os.path.join(self.tmpdirname							,			"tokenizer.bin"				)
						with open(A__							,			"wb"				) as handle:
										pickle.dump(A__							,			A__				)

						with open(A__							,			"rb"				) as handle:
										lowercase         = pickle.load(A__				)

						lowercase         = tokenizer_new.tokenize(A__				)

						self.assertListEqual(A__							,			A__				)




		@require_jumanpp
		def 				UpperCAmelCase__   (self      :	int				)					-> int:
						lowercase         = JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,			)




		@require_jumanpp
		def 				UpperCAmelCase__   (self      :	Optional[int]				)					-> List[Any]:
						lowercase         = JumanppTokenizer(do_lower_case=A__				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,			)




		@require_jumanpp
		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> str:
						lowercase         = JumanppTokenizer(normalize_text=A__				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,			)




		@require_jumanpp
		def 				UpperCAmelCase__   (self      :	int				)					-> int:
						lowercase         = JumanppTokenizer(trim_whitespace=A__				)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "				)							,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]							,			)




		@require_jumanpp
		def 				UpperCAmelCase__   (self      :	Union[str, Any]				)					-> int:
						lowercase         = JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"				)							,			["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]							,			)




		def 				UpperCAmelCase__   (self      :	Any				)					-> Union[str, Any]:
						lowercase         = ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

						lowercase         = {}
						for i, token in enumerate(A__				):
										lowercase         = i
						lowercase         = WordpieceTokenizer(vocab=A__							,			unk_token="[UNK]"				)

						self.assertListEqual(tokenizer.tokenize(""				)							,			[]				)

						self.assertListEqual(tokenizer.tokenize("こんにちは"				)							,			["こんにちは"]				)

						self.assertListEqual(tokenizer.tokenize("こんばんは"				)							,			["こん", "##ばんは"]				)

						self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"				)							,			["こん", "##ばんは", "[UNK]", "こんにちは"]				)




		def 				UpperCAmelCase__   (self      :	Union[str, Any]				)					-> Any:
						lowercase         = BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"				)
						lowercase         = tokenizer.subword_tokenizer

						lowercase         = subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"				)
						self.assertListEqual(A__							,			["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]				)

						lowercase         = subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"				)
						self.assertListEqual(A__							,			["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]				)





		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> str:
						lowercase         = self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"				)

						lowercase         = tokenizer.encode("ありがとう。"							,			add_special_tokens=A__				)
						lowercase         = tokenizer.encode("どういたしまして。"							,			add_special_tokens=A__				)

						lowercase         = tokenizer.build_inputs_with_special_tokens(A__				)
						lowercase         = tokenizer.build_inputs_with_special_tokens(A__							,			A__				)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class       UpperCAmelCase (						_lowercase			,       unittest.TestCase  ):
		UpperCAmelCase					:	str 			=		BertJapaneseTokenizer
		UpperCAmelCase					:	Dict 			=		False




		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Tuple:
						super().setUp()

						lowercase         = ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

						lowercase         = os.path.join(self.tmpdirname							,			VOCAB_FILES_NAMES["vocab_file"]				)
						with open(self.vocab_file							,			"w"							,			encoding="utf-8"				) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)




		def 				UpperCAmelCase__   (self      :	Dict							,			**A__      :	Optional[int]				)					-> str:
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,			subword_tokenizer_type="character"							,			**A__				)




		def 				UpperCAmelCase__   (self      :	str							,			A__      :	Any				)					-> int:
						lowercase         = "こんにちは、世界。 \nこんばんは、世界。"
						lowercase         = "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
						return input_text, output_text




		def 				UpperCAmelCase__   (self      :	Optional[int]				)					-> int:
						pass  # TODO add if relevant




		def 				UpperCAmelCase__   (self      :	str				)					-> Optional[int]:
						pass  # TODO add if relevant




		def 				UpperCAmelCase__   (self      :	Tuple				)					-> Any:
						pass  # TODO add if relevant




		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Optional[int]:
						lowercase         = self.tokenizer_class(self.vocab_file							,			subword_tokenizer_type="character"				)

						lowercase         = tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"				)
						self.assertListEqual(
						    A__							,			["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]				)
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A__				)							,			[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]				)




		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Dict:
						lowercase         = ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

						lowercase         = {}
						for i, token in enumerate(A__				):
										lowercase         = i
						lowercase         = CharacterTokenizer(vocab=A__							,			unk_token="[UNK]"				)

						self.assertListEqual(tokenizer.tokenize(""				)							,			[]				)

						self.assertListEqual(tokenizer.tokenize("こんにちは"				)							,			["こ", "ん", "に", "ち", "は"]				)

						self.assertListEqual(tokenizer.tokenize("こんにちほ"				)							,			["こ", "ん", "に", "ち", "[UNK]"]				)





		def 				UpperCAmelCase__   (self      :	Tuple				)					-> List[Any]:
						lowercase         = self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"				)

						lowercase         = tokenizer.encode("ありがとう。"							,			add_special_tokens=A__				)
						lowercase         = tokenizer.encode("どういたしまして。"							,			add_special_tokens=A__				)

						lowercase         = tokenizer.build_inputs_with_special_tokens(A__				)
						lowercase         = tokenizer.build_inputs_with_special_tokens(A__							,			A__				)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class       UpperCAmelCase (						unittest.TestCase  ):




		def 				UpperCAmelCase__   (self      :	Dict				)					-> Optional[int]:
						lowercase         = "cl-tohoku/bert-base-japanese"
						lowercase         = AutoTokenizer.from_pretrained(A__				)
						self.assertIsInstance(A__							,			A__				)




class       UpperCAmelCase (						unittest.TestCase  ):




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Any:
						lowercase         = "cl-tohoku/bert-base-japanese"
						with self.assertLogs("transformers"							,			level="WARNING"				) as cm:
										BertTokenizer.from_pretrained(A__				)
										self.assertTrue(
										    cm.records[0].message.startswith(
										        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
										        " is called from."				)				)
						lowercase         = "bert-base-cased"
						with self.assertLogs("transformers"							,			level="WARNING"				) as cm:
										BertJapaneseTokenizer.from_pretrained(A__				)
										self.assertTrue(
										    cm.records[0].message.startswith(
										        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
										        " is called from."				)				)











'''simple docstring'''

import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Image
from .base import TaskTemplate

@dataclass(frozen=_lowercase  )
class       UpperCAmelCase (						_lowercase  ):
		UpperCAmelCase					:	str 			=		field(default='''image-classification'''			,       metadata={'''include_in_asdict_even_if_is_default''': True}  )
		UpperCAmelCase					:	ClassVar[Features] 			=		Features({'''image''': Image()}  )
		UpperCAmelCase					:	ClassVar[Features] 			=		Features({'''labels''': ClassLabel}  )
		UpperCAmelCase					:	str 			=		"image"
		UpperCAmelCase					:	str 			=		"labels"




		def 				UpperCAmelCase__   (self      :	List[str]							,			A__      :	Union[str, Any]				)					-> Union[str, Any]:
						if self.label_column not in features:
										raise ValueError(f'Column {self.label_column} is not present in features.'				)
						if not isinstance(features[self.label_column]							,			A__				):
										raise ValueError(f'Column {self.label_column} is not a ClassLabel.'				)
						lowercase         = copy.deepcopy(self				)
						lowercase         = self.label_schema.copy()
						lowercase         = features[self.label_column]
						lowercase         = label_schema
						return task_template





		@property
		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> Dict[str, str]:
						return {
						    self.image_column: "image",
						    self.label_column: "labels",
						}




