



'''simple docstring'''



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class _UpperCamelCase		(     A      ,       unittest.TestCase		):





				'''simple docstring'''





				lowerCAmelCase__					    =    BertTokenizer
				lowerCAmelCase__					    =    BertTokenizerFast
				lowerCAmelCase__					    =    True
				lowerCAmelCase__					    =    True
				lowerCAmelCase__					    =    filter_non_english

				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											super().setUp()

											__lowercase       	=[
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											__lowercase       	=os.path.join(self.tmpdirname  ,       VOCAB_FILES_NAMES['vocab_file'])
											with open(self.vocab_file  ,       'w'  ,       encoding='utf-8') as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

				def 				__lowerCamelCase     (							self  :  List[str]  ,       _lowerCAmelCase  :  Dict):







											'''simple docstring'''

											__lowercase       	='UNwant\u00E9d,running'
											__lowercase       	='unwanted, running'
											return input_text, output_text

				def 				__lowerCamelCase     (							self  :  Optional[int]):







											'''simple docstring'''

											__lowercase       	=self.tokenizer_class(self.vocab_file)

											__lowercase       	=tokenizer.tokenize('UNwant\u00E9d,running')
											self.assertListEqual(_lowerCAmelCase  ,       ['un', '##want', '##ed', ',', 'runn', '##ing'])
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase)  ,       [9, 6, 7, 1_2, 1_0, 1_1])

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											if not self.test_rust_tokenizer:
																		return

											__lowercase       	=self.get_tokenizer()
											__lowercase       	=self.get_rust_tokenizer()

											__lowercase       	='UNwant\u00E9d,running'

											__lowercase       	=tokenizer.tokenize(_lowerCAmelCase)
											__lowercase       	=rust_tokenizer.tokenize(_lowerCAmelCase)
											self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

											__lowercase       	=tokenizer.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)
											__lowercase       	=rust_tokenizer.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)
											self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

											__lowercase       	=self.get_rust_tokenizer()
											__lowercase       	=tokenizer.encode(_lowerCAmelCase)
											__lowercase       	=rust_tokenizer.encode(_lowerCAmelCase)
											self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

											# With lower casing
											__lowercase       	=self.get_tokenizer(do_lower_case=_lowerCAmelCase)
											__lowercase       	=self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase)

											__lowercase       	='UNwant\u00E9d,running'

											__lowercase       	=tokenizer.tokenize(_lowerCAmelCase)
											__lowercase       	=rust_tokenizer.tokenize(_lowerCAmelCase)
											self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

											__lowercase       	=tokenizer.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)
											__lowercase       	=rust_tokenizer.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)
											self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

											__lowercase       	=self.get_rust_tokenizer()
											__lowercase       	=tokenizer.encode(_lowerCAmelCase)
											__lowercase       	=rust_tokenizer.encode(_lowerCAmelCase)
											self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  Optional[int]):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')  ,       ['ah', '\u535A', '\u63A8', 'zz'])

				def 				__lowerCamelCase     (							self  :  int):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')  ,       ['hello', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,       ['hello'])

				def 				__lowerCamelCase     (							self  :  int):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase  ,       strip_accents=_lowerCAmelCase)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,       ['hällo', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,       ['h\u00E9llo'])

				def 				__lowerCamelCase     (							self  :  List[Any]):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase  ,       strip_accents=_lowerCAmelCase)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,       ['hallo', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,       ['hello'])

				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,       ['hallo', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,       ['hello'])

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')  ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?'])

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase  ,       strip_accents=_lowerCAmelCase)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,       ['HäLLo', '!', 'how', 'Are', 'yoU', '?'])

				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase  ,       strip_accents=_lowerCAmelCase)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,       ['HaLLo', '!', 'how', 'Are', 'yoU', '?'])

				def 				__lowerCamelCase     (							self  :  Tuple):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer(do_lower_case=_lowerCAmelCase  ,       never_split=['[UNK]'])

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')  ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])

				def 				__lowerCamelCase     (							self  :  Optional[int]):







											'''simple docstring'''

											__lowercase       	=BasicTokenizer()
											__lowercase       	='a\n\'ll !!to?\'d of, can\'t.'
											__lowercase       	=['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
											self.assertListEqual(tokenizer.tokenize(_lowerCAmelCase)  ,       _lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[Any]):







											'''simple docstring'''

											__lowercase       	=['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											__lowercase       	={}
											for i, token in enumerate(_lowerCAmelCase):
																		__lowercase       	=i
											__lowercase       	=WordpieceTokenizer(vocab=_lowerCAmelCase  ,       unk_token='[UNK]')

											self.assertListEqual(tokenizer.tokenize('')  ,       [])

											self.assertListEqual(tokenizer.tokenize('unwanted running')  ,       ['un', '##want', '##ed', 'runn', '##ing'])

											self.assertListEqual(tokenizer.tokenize('unwantedX running')  ,       ['[UNK]', 'runn', '##ing'])

				def 				__lowerCamelCase     (							self  :  Tuple):







											'''simple docstring'''

											self.assertTrue(_is_whitespace(' '))
											self.assertTrue(_is_whitespace('\t'))
											self.assertTrue(_is_whitespace('\r'))
											self.assertTrue(_is_whitespace('\n'))
											self.assertTrue(_is_whitespace('\u00A0'))

											self.assertFalse(_is_whitespace('A'))
											self.assertFalse(_is_whitespace('-'))

				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											self.assertTrue(_is_control('\u0005'))

											self.assertFalse(_is_control('A'))
											self.assertFalse(_is_control(' '))
											self.assertFalse(_is_control('\t'))
											self.assertFalse(_is_control('\r'))

				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											self.assertTrue(_is_punctuation('-'))
											self.assertTrue(_is_punctuation('$'))
											self.assertTrue(_is_punctuation('`'))
											self.assertTrue(_is_punctuation('.'))

											self.assertFalse(_is_punctuation('A'))
											self.assertFalse(_is_punctuation(' '))

				def 				__lowerCamelCase     (							self  :  Tuple):







											'''simple docstring'''

											__lowercase       	=self.get_tokenizer()
											__lowercase       	=self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase) for t in ['Test', '\xad', 'test']]  ,       [['[UNK]'], [], ['[UNK]']])

											self.assertListEqual(
											    [rust_tokenizer.tokenize(_lowerCAmelCase) for t in ['Test', '\xad', 'test']]  ,       [['[UNK]'], [], ['[UNK]']])

				@slow
				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											__lowercase       	=self.tokenizer_class.from_pretrained('bert-base-uncased')

											__lowercase       	=tokenizer.encode('sequence builders'  ,       add_special_tokens=_lowerCAmelCase)
											__lowercase       	=tokenizer.encode('multi-sequence build'  ,       add_special_tokens=_lowerCAmelCase)

											__lowercase       	=tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase)
											__lowercase       	=tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase  ,       _lowerCAmelCase)

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

				def 				__lowerCamelCase     (							self  :  int):







											'''simple docstring'''

											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																									__lowercase       	=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,       **_lowerCAmelCase)

																									__lowercase       	=f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																									__lowercase       	=tokenizer_r.encode_plus(
																									    _lowerCAmelCase  ,       return_attention_mask=_lowerCAmelCase  ,       return_token_type_ids=_lowerCAmelCase  ,       return_offsets_mapping=_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase  ,       )

																									__lowercase       	=tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase  ,       'do_lower_case') else False
																									__lowercase       	=(
																									    [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), 'A'),
																									        ((1, 2), ','),
																									        ((3, 5), 'na'),
																									        ((5, 6), '##ï'),
																									        ((6, 8), '##ve'),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), 'Allen'),
																									        ((2_1, 2_3), '##NL'),
																									        ((2_3, 2_4), '##P'),
																									        ((2_5, 3_3), 'sentence'),
																									        ((3_3, 3_4), '.'),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									    if not do_lower_case
																									    else [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), 'a'),
																									        ((1, 2), ','),
																									        ((3, 8), 'naive'),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), 'allen'),
																									        ((2_1, 2_3), '##nl'),
																									        ((2_3, 2_4), '##p'),
																									        ((2_5, 3_3), 'sentence'),
																									        ((3_3, 3_4), '.'),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									)

																									self.assertEqual(
																									    [e[1] for e in expected_results]  ,       tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
																									self.assertEqual([e[0] for e in expected_results]  ,       tokens['offset_mapping'])

				def 				__lowerCamelCase     (							self  :  int):







											'''simple docstring'''

											__lowercase       	=['的', '人', '有']
											__lowercase       	=''.join(_lowerCAmelCase)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																									__lowercase       	=True
																									__lowercase       	=self.tokenizer_class.from_pretrained(_lowerCAmelCase  ,       **_lowerCAmelCase)
																									__lowercase       	=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,       **_lowerCAmelCase)

																									__lowercase       	=tokenizer_p.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)
																									__lowercase       	=tokenizer_r.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)

																									__lowercase       	=tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase)
																									__lowercase       	=tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase)

																									# it is expected that each Chinese character is not preceded by "##"
																									self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)
																									self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

																									__lowercase       	=False
																									__lowercase       	=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,       **_lowerCAmelCase)
																									__lowercase       	=self.tokenizer_class.from_pretrained(_lowerCAmelCase  ,       **_lowerCAmelCase)

																									__lowercase       	=tokenizer_r.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)
																									__lowercase       	=tokenizer_p.encode(_lowerCAmelCase  ,       add_special_tokens=_lowerCAmelCase)

																									__lowercase       	=tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase)
																									__lowercase       	=tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase)

																									# it is expected that only the first Chinese character is not preceded by "##".
																									__lowercase       	=[
																									    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase)
																									]
																									self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)
																									self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)








'''simple docstring'''



# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowerCamelCase	 =       """Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"""



def   _A       (   ):



							"""simple docstring"""



							__lowercase       	=_ask_options(
							    'In which compute environment are you running?'  ,   ['This machine', 'AWS (Amazon SageMaker)']  ,   _convert_compute_environment  ,   )
							if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
														__lowercase       	=get_sagemaker_input()
							else:
														__lowercase       	=get_cluster_input()
							return config



def   _A       (   _lowerCAmelCase=None					):



							"""simple docstring"""



							if subparsers is not None:
														__lowercase       	=subparsers.add_parser('config'  ,   description=_lowerCAmelCase					)
							else:
														__lowercase       	=argparse.ArgumentParser('Accelerate config command'  ,   description=_lowerCAmelCase					)

							parser.add_argument(
							    '--config_file'  ,   default=_lowerCAmelCase  ,   help=(
							        'The path to use to store the config file. Will default to a file named default_config.yaml in the cache '
							        'location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '
							        'such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '
							        'with \'huggingface\'.'
							    )  ,   )

							if subparsers is not None:
														parser.set_defaults(func=_lowerCAmelCase					)
							return parser



def   _A       (   _lowerCAmelCase					):



							"""simple docstring"""



							__lowercase       	=get_user_input()
							if args.config_file is not None:
														__lowercase       	=args.config_file
							else:
														if not os.path.isdir(_lowerCAmelCase					):
																					os.makedirs(_lowerCAmelCase					)
														__lowercase       	=default_yaml_config_file

							if config_file.endswith('.json'					):
														config.to_json_file(_lowerCAmelCase					)
							else:
														config.to_yaml_file(_lowerCAmelCase					)
							print(f"""accelerate configuration saved at {config_file}"""					)



def   _A       (   ):



							"""simple docstring"""



							__lowercase       	=config_command_parser()
							__lowercase       	=parser.parse_args()
							config_command(_lowerCAmelCase					)


if __name__ == "__main__":
		main()



