






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class  SCREAMING_SNAKE_CASE			(			a__      ,		unittest.TestCase					):
     '''simple docstring'''


     UpperCamelCase_							:		Tuple			=	BertJapaneseTokenizer
     UpperCamelCase_							:		Any			=	False
     UpperCamelCase_							:		Tuple			=	True


     def 				_A       (				self       :	Optional[int]      ):
       super().setUp()

       SCREAMING_SNAKE_CASE       :      Tuple							=    [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           'こんにちは',
           'こん',
           'にちは',
           'ばんは',
           '##こん',
           '##にちは',
           '##ばんは',
           '世界',
           '##世界',
           '、',
           '##、',
           '。',
           '##。',
       ]

       SCREAMING_SNAKE_CASE       :      str							=    os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file					, "w"					, encoding="utf-8"      ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )


     def 				_A       (				self       :	str					, UpperCAmelCase_       :	List[str]      ):
       SCREAMING_SNAKE_CASE       :      Tuple							=    'こんにちは、世界。 \nこんばんは、世界。'
       SCREAMING_SNAKE_CASE       :      List[str]							=    'こんにちは 、 世界 。 こんばんは 、 世界 。'
       return input_text, output_text


     def 				_A       (				self       :	Any					, UpperCAmelCase_       :	Any      ):
       SCREAMING_SNAKE_CASE       :      List[Any]							=    self.get_input_output_texts(_A      )
       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer.encode(_A					, add_special_tokens=_A      )
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tokenizer.decode(_A					, clean_up_tokenization_spaces=_A      )
       return text, ids


     def 				_A       (				self       :	Any      ):
       pass  # TODO add if relevant


     def 				_A       (				self       :	Dict      ):
       pass  # TODO add if relevant


     def 				_A       (				self       :	List[str]      ):
       pass  # TODO add if relevant


     def 				_A       (				self       :	Tuple      ):
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    self.tokenizer_class(self.vocab_file      )

       SCREAMING_SNAKE_CASE       :      Tuple							=    tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"      )
       self.assertListEqual(_A					, ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )					, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )


     def 				_A       (				self       :	Dict      ):
       SCREAMING_SNAKE_CASE       :      str							=    self.tokenizer_class(self.vocab_file					, word_tokenizer_type="mecab"      )
       self.assertIsNotNone(_A      )

       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    'こんにちは、世界。\nこんばんは、世界。'
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    tokenizer.tokenize(_A      )
       self.assertListEqual(_A					, ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )					, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    os.path.join(self.tmpdirname					, "tokenizer.bin"      )
       with open(_A					, "wb"      ) as handle:
         pickle.dump(_A					, _A      )

       with open(_A					, "rb"      ) as handle:
         SCREAMING_SNAKE_CASE       :      Optional[int]							=    pickle.load(_A      )

       SCREAMING_SNAKE_CASE       :      Optional[int]							=    tokenizer_new.tokenize(_A      )

       self.assertListEqual(_A					, _A      )


     def 				_A       (				self       :	List[Any]      ):
       SCREAMING_SNAKE_CASE       :      Tuple							=    MecabTokenizer(mecab_dic="ipadic"      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					, )


     def 				_A       (				self       :	Optional[Any]      ):

       try:
         SCREAMING_SNAKE_CASE       :      Any							=    MecabTokenizer(mecab_dic="unidic_lite"      )
       except ModuleNotFoundError:
         return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					, )


     def 				_A       (				self       :	Optional[int]      ):

       try:
         SCREAMING_SNAKE_CASE       :      int							=    MecabTokenizer(mecab_dic="unidic"      )
       except ModuleNotFoundError:
         return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					, )


     def 				_A       (				self       :	Any      ):
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    MecabTokenizer(do_lower_case=_A					, mecab_dic="ipadic"      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]					, )


     def 				_A       (				self       :	Dict      ):

       try:
         SCREAMING_SNAKE_CASE       :      List[str]							=    MecabTokenizer(
             do_lower_case=_A					, normalize_text=_A					, mecab_option="-d /usr/local/lib/mecab/dic/jumandic"      )
       except RuntimeError:
         # if dict doesn't exist in the system, previous code raises this error.
         return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]					, )


     def 				_A       (				self       :	Optional[Any]      ):
       SCREAMING_SNAKE_CASE       :      str							=    MecabTokenizer(normalize_text=_A					, mecab_dic="ipadic"      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]					, )


     @require_sudachi
     def 				_A       (				self       :	Optional[Any]      ):
       SCREAMING_SNAKE_CASE       :      List[Any]							=    self.tokenizer_class(self.vocab_file					, word_tokenizer_type="sudachi"      )
       self.assertIsNotNone(_A      )

       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    'こんにちは、世界。\nこんばんは、世界。'
       SCREAMING_SNAKE_CASE       :      str							=    tokenizer.tokenize(_A      )
       self.assertListEqual(_A					, ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )					, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

       SCREAMING_SNAKE_CASE       :      Optional[int]							=    os.path.join(self.tmpdirname					, "tokenizer.bin"      )
       with open(_A					, "wb"      ) as handle:
         pickle.dump(_A					, _A      )

       with open(_A					, "rb"      ) as handle:
         SCREAMING_SNAKE_CASE       :      Any							=    pickle.load(_A      )

       SCREAMING_SNAKE_CASE       :      Tuple							=    tokenizer_new.tokenize(_A      )

       self.assertListEqual(_A					, _A      )


     @require_sudachi
     def 				_A       (				self       :	Tuple      ):
       SCREAMING_SNAKE_CASE       :      Tuple							=    SudachiTokenizer(sudachi_dict_type="core"      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]					, )


     @require_sudachi
     def 				_A       (				self       :	Union[str, Any]      ):
       SCREAMING_SNAKE_CASE       :      Dict							=    SudachiTokenizer(sudachi_dict_type="core"					, sudachi_split_mode="A"      )

       self.assertListEqual(tokenizer.tokenize("外国人参政権"      )					, ["外国", "人", "参政", "権"]      )


     @require_sudachi
     def 				_A       (				self       :	List[Any]      ):
       SCREAMING_SNAKE_CASE       :      List[Any]							=    SudachiTokenizer(sudachi_dict_type="core"					, sudachi_split_mode="B"      )

       self.assertListEqual(tokenizer.tokenize("外国人参政権"      )					, ["外国人", "参政権"]      )


     @require_sudachi
     def 				_A       (				self       :	int      ):
       SCREAMING_SNAKE_CASE       :      int							=    SudachiTokenizer(sudachi_dict_type="core"					, sudachi_split_mode="C"      )

       self.assertListEqual(tokenizer.tokenize("外国人参政権"      )					, ["外国人参政権"]      )


     @require_sudachi
     def 				_A       (				self       :	Union[str, Any]      ):
       SCREAMING_SNAKE_CASE       :      Tuple							=    SudachiTokenizer(do_lower_case=_A					, sudachi_dict_type="core"      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]					, )


     @require_sudachi
     def 				_A       (				self       :	Union[str, Any]      ):
       SCREAMING_SNAKE_CASE       :      Any							=    SudachiTokenizer(normalize_text=_A					, sudachi_dict_type="core"      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]					, )


     @require_sudachi
     def 				_A       (				self       :	Any      ):
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    SudachiTokenizer(trim_whitespace=_A					, sudachi_dict_type="core"      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					, )


     @require_jumanpp
     def 				_A       (				self       :	Optional[int]      ):
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    self.tokenizer_class(self.vocab_file					, word_tokenizer_type="jumanpp"      )
       self.assertIsNotNone(_A      )

       SCREAMING_SNAKE_CASE       :      Dict							=    'こんにちは、世界。\nこんばんは、世界。'
       SCREAMING_SNAKE_CASE       :      List[Any]							=    tokenizer.tokenize(_A      )
       self.assertListEqual(_A					, ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )					, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    os.path.join(self.tmpdirname					, "tokenizer.bin"      )
       with open(_A					, "wb"      ) as handle:
         pickle.dump(_A					, _A      )

       with open(_A					, "rb"      ) as handle:
         SCREAMING_SNAKE_CASE       :      str							=    pickle.load(_A      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer_new.tokenize(_A      )

       self.assertListEqual(_A					, _A      )


     @require_jumanpp
     def 				_A       (				self       :	Optional[Any]      ):
       SCREAMING_SNAKE_CASE       :      Any							=    JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					, )


     @require_jumanpp
     def 				_A       (				self       :	int      ):
       SCREAMING_SNAKE_CASE       :      Dict							=    JumanppTokenizer(do_lower_case=_A      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					, )


     @require_jumanpp
     def 				_A       (				self       :	Any      ):
       SCREAMING_SNAKE_CASE       :      Any							=    JumanppTokenizer(normalize_text=_A      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					, )


     @require_jumanpp
     def 				_A       (				self       :	int      ):
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    JumanppTokenizer(trim_whitespace=_A      )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )					, ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]					, )


     @require_jumanpp
     def 				_A       (				self       :	Any      ):
       SCREAMING_SNAKE_CASE       :      List[str]							=    JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"      )					, ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]					, )


     def 				_A       (				self       :	Optional[Any]      ):
       SCREAMING_SNAKE_CASE       :      List[str]							=    ['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

       SCREAMING_SNAKE_CASE       :      Any							=    {}
       for i, token in enumerate(_A      ):
         SCREAMING_SNAKE_CASE       :      int							=    i
       SCREAMING_SNAKE_CASE       :      Tuple							=    WordpieceTokenizer(vocab=_A					, unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      )					, []      )

       self.assertListEqual(tokenizer.tokenize("こんにちは"      )					, ["こんにちは"]      )

       self.assertListEqual(tokenizer.tokenize("こんばんは"      )					, ["こん", "##ばんは"]      )

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"      )					, ["こん", "##ばんは", "[UNK]", "こんにちは"]      )


     def 				_A       (				self       :	Dict      ):
       SCREAMING_SNAKE_CASE       :      Any							=    BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"      )
       SCREAMING_SNAKE_CASE       :      int							=    tokenizer.subword_tokenizer

       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"      )
       self.assertListEqual(_A					, ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]      )

       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"      )
       self.assertListEqual(_A					, ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]      )






     def 				_A       (				self       :	Optional[Any]      ):
       SCREAMING_SNAKE_CASE       :      List[Any]							=    self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer.encode("ありがとう。"					, add_special_tokens=_A      )
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tokenizer.encode("どういたしまして。"					, add_special_tokens=_A      )

       SCREAMING_SNAKE_CASE       :      Tuple							=    tokenizer.build_inputs_with_special_tokens(_A      )
       SCREAMING_SNAKE_CASE       :      Tuple							=    tokenizer.build_inputs_with_special_tokens(_A					, _A      )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  SCREAMING_SNAKE_CASE			(			a__      ,		unittest.TestCase					):
     '''simple docstring'''


     UpperCamelCase_							:		Optional[int]			=	BertJapaneseTokenizer
     UpperCamelCase_							:		int			=	False


     def 				_A       (				self       :	str      ):
       super().setUp()

       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

       SCREAMING_SNAKE_CASE       :      Tuple							=    os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file					, "w"					, encoding="utf-8"      ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )


     def 				_A       (				self       :	Dict					, **UpperCAmelCase_       :	Dict      ):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					, subword_tokenizer_type="character"					, **_A      )


     def 				_A       (				self       :	Optional[Any]					, UpperCAmelCase_       :	Dict      ):
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    'こんにちは、世界。 \nこんばんは、世界。'
       SCREAMING_SNAKE_CASE       :      int							=    'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
       return input_text, output_text


     def 				_A       (				self       :	List[Any]      ):
       pass  # TODO add if relevant


     def 				_A       (				self       :	List[Any]      ):
       pass  # TODO add if relevant


     def 				_A       (				self       :	List[Any]      ):
       pass  # TODO add if relevant


     def 				_A       (				self       :	List[str]      ):
       SCREAMING_SNAKE_CASE       :      Tuple							=    self.tokenizer_class(self.vocab_file					, subword_tokenizer_type="character"      )

       SCREAMING_SNAKE_CASE       :      Any							=    tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"      )
       self.assertListEqual(
           _A					, ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]      )
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_A      )					, [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]      )


     def 				_A       (				self       :	Tuple      ):
       SCREAMING_SNAKE_CASE       :      List[Any]							=    ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

       SCREAMING_SNAKE_CASE       :      Tuple							=    {}
       for i, token in enumerate(_A      ):
         SCREAMING_SNAKE_CASE       :      int							=    i
       SCREAMING_SNAKE_CASE       :      Dict							=    CharacterTokenizer(vocab=_A					, unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      )					, []      )

       self.assertListEqual(tokenizer.tokenize("こんにちは"      )					, ["こ", "ん", "に", "ち", "は"]      )

       self.assertListEqual(tokenizer.tokenize("こんにちほ"      )					, ["こ", "ん", "に", "ち", "[UNK]"]      )






     def 				_A       (				self       :	Optional[int]      ):
       SCREAMING_SNAKE_CASE       :      List[Any]							=    self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer.encode("ありがとう。"					, add_special_tokens=_A      )
       SCREAMING_SNAKE_CASE       :      Dict							=    tokenizer.encode("どういたしまして。"					, add_special_tokens=_A      )

       SCREAMING_SNAKE_CASE       :      int							=    tokenizer.build_inputs_with_special_tokens(_A      )
       SCREAMING_SNAKE_CASE       :      Any							=    tokenizer.build_inputs_with_special_tokens(_A					, _A      )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  SCREAMING_SNAKE_CASE			(			unittest.TestCase					):
     '''simple docstring'''


     def 				_A       (				self       :	Tuple      ):
       SCREAMING_SNAKE_CASE       :      List[str]							=    'cl-tohoku/bert-base-japanese'
       SCREAMING_SNAKE_CASE       :      int							=    AutoTokenizer.from_pretrained(_A      )
       self.assertIsInstance(_A					, _A      )





class  SCREAMING_SNAKE_CASE			(			unittest.TestCase					):
     '''simple docstring'''


     def 				_A       (				self       :	int      ):
       SCREAMING_SNAKE_CASE       :      Any							=    'cl-tohoku/bert-base-japanese'
       with self.assertLogs("transformers"					, level="WARNING"      ) as cm:
         BertTokenizer.from_pretrained(_A      )
         self.assertTrue(
             cm.records[0].message.startswith(
                 "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                 " is called from."      )      )
       SCREAMING_SNAKE_CASE       :      Tuple							=    'bert-base-cased'
       with self.assertLogs("transformers"					, level="WARNING"      ) as cm:
         BertJapaneseTokenizer.from_pretrained(_A      )
         self.assertTrue(
             cm.records[0].message.startswith(
                 "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                 " is called from."      )      )














# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path

import torch

from ...utils import is_npu_available, is_xpu_available
from .config_args import ClusterConfig, default_json_config_file
from .config_utils import SubcommandHelpFormatter


snake_case    				=		"""Create a default config file for Accelerate with only a few flags set."""


def   lowerCamelCase__			(     lowercase="no"						,     lowercase = default_json_config_file						,     lowercase = False				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Tuple							=    Path(lowercase				)
  path.parent.mkdir(parents=lowercase						,     exist_ok=lowercase				)
  if path.exists():
    print(
        F'''Configuration already exists at {save_location}, will not override. Run `accelerate config` manually or pass a different `save_location`.'''				)
    return False
  SCREAMING_SNAKE_CASE       :      int							=    mixed_precision.lower()
  if mixed_precision not in ["no", "fp16", "bf16", "fp8"]:
    raise ValueError(
        F'''`mixed_precision` should be one of \'no\', \'fp16\', \'bf16\', or \'fp8\'. Received {mixed_precision}'''				)
  SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    {
      "compute_environment": "LOCAL_MACHINE",
      "mixed_precision": mixed_precision,
  }
  if torch.cuda.is_available():
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    torch.cuda.device_count()
    SCREAMING_SNAKE_CASE       :      int							=    num_gpus
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    False
    if num_gpus > 1:
      SCREAMING_SNAKE_CASE       :      Tuple							=    "MULTI_GPU"
    else:
      SCREAMING_SNAKE_CASE       :      Optional[Any]							=    "NO"
  elif is_xpu_available() and use_xpu:
    SCREAMING_SNAKE_CASE       :      List[str]							=    torch.xpu.device_count()
    SCREAMING_SNAKE_CASE       :      str							=    num_xpus
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    False
    if num_xpus > 1:
      SCREAMING_SNAKE_CASE       :      Any							=    "MULTI_XPU"
    else:
      SCREAMING_SNAKE_CASE       :      str							=    "NO"
  elif is_npu_available():
    SCREAMING_SNAKE_CASE       :      List[Any]							=    torch.npu.device_count()
    SCREAMING_SNAKE_CASE       :      Optional[Any]							=    num_npus
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    False
    if num_npus > 1:
      SCREAMING_SNAKE_CASE       :      str							=    "MULTI_NPU"
    else:
      SCREAMING_SNAKE_CASE       :      int							=    "NO"
  else:
    SCREAMING_SNAKE_CASE       :      Dict							=    0
    SCREAMING_SNAKE_CASE       :      List[Any]							=    True
    SCREAMING_SNAKE_CASE       :      Dict							=    1
    SCREAMING_SNAKE_CASE       :      int							=    "NO"
  SCREAMING_SNAKE_CASE       :      Dict							=    ClusterConfig(**lowercase				)
  config.to_json_file(lowercase				)
  return path


def   lowerCamelCase__			(     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Dict							=    parser.add_parser("default"						,     parents=lowercase						,     help=lowercase						,     formatter_class=lowercase				)
  parser.add_argument(
      "--config_file"						,     default=lowercase						,     help=(
          "The path to use to store the config file. Will default to a file named default_config.yaml in the cache "
          "location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have "
          "such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed "
          "with 'huggingface'."
      )						,     dest="save_location"						,     )

  parser.add_argument(
      "--mixed_precision"						,     choices=["no", "fp16", "bf16"]						,     type=lowercase						,     help="Whether or not to use mixed precision training. "
      "Choose between FP16 and BF16 (bfloat16) training. "
      "BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later."						,     default="no"						,     )
  parser.set_defaults(func=lowercase				)
  return parser


def   lowerCamelCase__			(     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Optional[int]							=    write_basic_config(args.mixed_precision						,     args.save_location				)
  if config_file:
    print(F'''accelerate configuration saved at {config_file}'''				)






