


'''simple docstring'''



import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin






class  UpperCamelCase__(     lowerCAmelCase		,       unittest.TestCase ):
				__magic_name__       :   Optional[int]							= ProphetNetTokenizer
				__magic_name__       :   List[str]							= False




				def   a__(	self    :				Dict   )->  Union[str, Any]:

										"""simple docstring"""




										super().setUp()

										UpperCAmelCase         =			[
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''[PAD]''',
										    '''[MASK]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										UpperCAmelCase         =			os.path.join(self.tmpdirname   ,					VOCAB_FILES_NAMES['''vocab_file''']   )
										with open(self.vocab_file   ,					'''w'''   ,					encoding='''utf-8'''   ) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )




				def   a__(	self    :				Tuple   ,					lowerCAmelCase    :				List[Any]   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			'''UNwant\u00E9d,running'''
										UpperCAmelCase         =			'''unwanted, running'''
										return input_text, output_text




				def   a__(	self    :				int   )->  List[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			self.tokenizer_class(self.vocab_file   )

										UpperCAmelCase         =			tokenizer.tokenize('''UNwant\u00E9d,running'''   )
										self.assertListEqual(lowerCAmelCase   ,					['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase   )   ,					[9, 6, 7, 12, 10, 11]   )




				def   a__(	self    :				int   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )   ,					['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )




				def   a__(	self    :				Tuple   )->  int:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )   ,					['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''hello''']   )




				def   a__(	self    :				Dict   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   ,					strip_accents=lowerCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''h\u00E9llo''']   )




				def   a__(	self    :				Any   )->  Any:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   ,					strip_accents=lowerCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''hello''']   )




				def   a__(	self    :				Any   )->  int:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''hello''']   )




				def   a__(	self    :				Optional[int]   )->  Any:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )   ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def   a__(	self    :				List[str]   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   ,					strip_accents=lowerCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def   a__(	self    :				Optional[int]   )->  List[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   ,					strip_accents=lowerCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def   a__(	self    :				Dict   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=lowerCAmelCase   ,					never_split=['''[UNK]''']   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )   ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )




				def   a__(	self    :				Optional[Any]   )->  int:

										"""simple docstring"""




										UpperCAmelCase         =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

										UpperCAmelCase         =			{}
										for i, token in enumerate(lowerCAmelCase   ):
																UpperCAmelCase         =			i
										UpperCAmelCase         =			WordpieceTokenizer(vocab=lowerCAmelCase   ,					unk_token='''[UNK]'''   )

										self.assertListEqual(tokenizer.tokenize(''''''   )   ,					[]   )

										self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )   ,					['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

										self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )   ,					['''[UNK]''', '''runn''', '''##ing''']   )




				@require_torch
				def   a__(	self    :				Any   )->  Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''   )

										UpperCAmelCase         =			['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
										UpperCAmelCase         =			[1037, 2146, 20423, 2005, 7680, 7849, 3989, 1012, 102]
										UpperCAmelCase         =			tokenizer(lowerCAmelCase   ,					padding=lowerCAmelCase   ,					return_tensors='''pt'''   )
										self.assertIsInstance(lowerCAmelCase   ,					lowerCAmelCase   )
										UpperCAmelCase         =			list(batch.input_ids.numpy()[0]   )
										self.assertListEqual(lowerCAmelCase   ,					lowerCAmelCase   )

										self.assertEqual((2, 9)   ,					batch.input_ids.shape   )
										self.assertEqual((2, 9)   ,					batch.attention_mask.shape   )




				def   a__(	self    :				int   )->  Tuple:

										"""simple docstring"""




										self.assertTrue(_is_whitespace(''' '''   )   )
										self.assertTrue(_is_whitespace('''\t'''   )   )
										self.assertTrue(_is_whitespace('''\r'''   )   )
										self.assertTrue(_is_whitespace('''\n'''   )   )
										self.assertTrue(_is_whitespace('''\u00A0'''   )   )

										self.assertFalse(_is_whitespace('''A'''   )   )
										self.assertFalse(_is_whitespace('''-'''   )   )




				def   a__(	self    :				str   )->  int:

										"""simple docstring"""




										self.assertTrue(_is_control('''\u0005'''   )   )

										self.assertFalse(_is_control('''A'''   )   )
										self.assertFalse(_is_control(''' '''   )   )
										self.assertFalse(_is_control('''\t'''   )   )
										self.assertFalse(_is_control('''\r'''   )   )




				def   a__(	self    :				Tuple   )->  Any:

										"""simple docstring"""




										self.assertTrue(_is_punctuation('''-'''   )   )
										self.assertTrue(_is_punctuation('''$'''   )   )
										self.assertTrue(_is_punctuation('''`'''   )   )
										self.assertTrue(_is_punctuation('''.'''   )   )

										self.assertFalse(_is_punctuation('''A'''   )   )
										self.assertFalse(_is_punctuation(''' '''   )   )





				@slow
				def   a__(	self    :				Tuple   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''   )

										UpperCAmelCase         =			tokenizer.encode('''sequence builders'''   ,					add_special_tokens=lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.encode('''multi-sequence build'''   ,					add_special_tokens=lowerCAmelCase   )

										UpperCAmelCase         =			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase   ,					lowerCAmelCase   )

										assert encoded_sentence == text + [102]
										assert encoded_pair == text + [102] + text_a + [102]









'''simple docstring'''



# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


_lowercase    :    Union[str, Any]			     =  """3"""

print("""Python version:""", sys.version)
print("""transformers version:""", transformers.__version__)

try:
	import torch

	print("""Torch version:""", torch.__version__)
	print("""Cuda available:""", torch.cuda.is_available())
	print("""Cuda version:""", torch.version.cuda)
	print("""CuDNN version:""", torch.backends.cudnn.version())
	print("""Number of GPUs available:""", torch.cuda.device_count())
	print("""NCCL version:""", torch.cuda.nccl.version())
except ImportError:
	print("""Torch version:""", None)

try:
	import deepspeed

	print("""DeepSpeed version:""", deepspeed.__version__)
except ImportError:
	print("""DeepSpeed version:""", None)

try:
	import tensorflow as tf

	print("""TensorFlow version:""", tf.__version__)
	print("""TF GPUs available:""", bool(tf.config.list_physical_devices("""GPU""")))
	print("""Number of TF GPUs available:""", len(tf.config.list_physical_devices("""GPU""")))
except ImportError:
	print("""TensorFlow version:""", None)





