import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      __lowercase					(    _lowerCamelCase       ,     unittest.TestCase    ):
						lowercase					 =     MobileBertTokenizer
						lowercase					 =     MobileBertTokenizerFast
						lowercase					 =     True
						lowercase					 =     True
						lowercase					 =     filter_non_english
						lowercase					 =     'google/mobilebert-uncased'

						def        __a		(	self				:      Any   )							->		Any:


									'''simple docstring'''

									super().setUp()

									lowercase							=		[
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									lowercase							=		os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['''vocab_file''']   )
									with open(self.vocab_file							,							'''w'''							,							encoding='''utf-8'''   ) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )

									lowercase							=		[
									    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
									    for tokenizer_def in self.tokenizers_list
									]

						def        __a		(	self				:      Optional[Any]							,							__lowerCamelCase				:      Union[str, Any]   )							->		List[str]:


									'''simple docstring'''

									lowercase							=		'''UNwant\u00E9d,running'''
									lowercase							=		'''unwanted, running'''
									return input_text, output_text

						def        __a		(	self				:      Union[str, Any]   )							->		Any:


									'''simple docstring'''

									lowercase							=		self.tokenizer_class(self.vocab_file   )

									lowercase							=		tokenizer.tokenize('''UNwant\u00E9d,running'''   )
									self.assertListEqual(A__							,							['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(A__   )							,							[9, 6, 7, 12, 10, 11]   )

						def        __a		(	self				:      List[str]   )							->		Optional[int]:


									'''simple docstring'''

									if not self.test_rust_tokenizer:
												return

									lowercase							=		self.get_tokenizer()
									lowercase							=		self.get_rust_tokenizer()

									lowercase							=		'''UNwant\u00E9d,running'''

									lowercase							=		tokenizer.tokenize(A__   )
									lowercase							=		rust_tokenizer.tokenize(A__   )
									self.assertListEqual(A__							,							A__   )

									lowercase							=		tokenizer.encode(A__							,							add_special_tokens=A__   )
									lowercase							=		rust_tokenizer.encode(A__							,							add_special_tokens=A__   )
									self.assertListEqual(A__							,							A__   )

									lowercase							=		self.get_rust_tokenizer()
									lowercase							=		tokenizer.encode(A__   )
									lowercase							=		rust_tokenizer.encode(A__   )
									self.assertListEqual(A__							,							A__   )

									# With lower casing
									lowercase							=		self.get_tokenizer(do_lower_case=A__   )
									lowercase							=		self.get_rust_tokenizer(do_lower_case=A__   )

									lowercase							=		'''UNwant\u00E9d,running'''

									lowercase							=		tokenizer.tokenize(A__   )
									lowercase							=		rust_tokenizer.tokenize(A__   )
									self.assertListEqual(A__							,							A__   )

									lowercase							=		tokenizer.encode(A__							,							add_special_tokens=A__   )
									lowercase							=		rust_tokenizer.encode(A__							,							add_special_tokens=A__   )
									self.assertListEqual(A__							,							A__   )

									lowercase							=		self.get_rust_tokenizer()
									lowercase							=		tokenizer.encode(A__   )
									lowercase							=		rust_tokenizer.encode(A__   )
									self.assertListEqual(A__							,							A__   )

						def        __a		(	self				:      Dict   )							->		Tuple:


									'''simple docstring'''

									lowercase							=		BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )							,							['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )

						def        __a		(	self				:      Dict   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )							,							['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,							['''hello''']   )

						def        __a		(	self				:      int   )							->		Optional[Any]:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__							,							strip_accents=A__   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,							['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,							['''h\u00E9llo''']   )

						def        __a		(	self				:      str   )							->		Any:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__							,							strip_accents=A__   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,							['''hello''']   )

						def        __a		(	self				:      Tuple   )							->		Any:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,							['''hello''']   )

						def        __a		(	self				:      Dict   )							->		str:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )							,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )

						def        __a		(	self				:      Dict   )							->		Dict:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__							,							strip_accents=A__   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,							['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )

						def        __a		(	self				:      int   )							->		Union[str, Any]:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__							,							strip_accents=A__   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,							['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )

						def        __a		(	self				:      Optional[int]   )							->		str:


									'''simple docstring'''

									lowercase							=		BasicTokenizer(do_lower_case=A__							,							never_split=['''[UNK]''']   )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )							,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )

						def        __a		(	self				:      Tuple   )							->		Tuple:


									'''simple docstring'''

									lowercase							=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									lowercase							=		{}
									for i, token in enumerate(A__   ):
												lowercase							=		i
									lowercase							=		WordpieceTokenizer(vocab=A__							,							unk_token='''[UNK]'''   )

									self.assertListEqual(tokenizer.tokenize(''''''   )							,							[]   )

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )							,							['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )							,							['''[UNK]''', '''runn''', '''##ing''']   )

						def        __a		(	self				:      List[Any]   )							->		str:


									'''simple docstring'''

									self.assertTrue(_is_whitespace(''' '''   )   )
									self.assertTrue(_is_whitespace('''\t'''   )   )
									self.assertTrue(_is_whitespace('''\r'''   )   )
									self.assertTrue(_is_whitespace('''\n'''   )   )
									self.assertTrue(_is_whitespace('''\u00A0'''   )   )

									self.assertFalse(_is_whitespace('''A'''   )   )
									self.assertFalse(_is_whitespace('''-'''   )   )

						def        __a		(	self				:      Any   )							->		Union[str, Any]:


									'''simple docstring'''

									self.assertTrue(_is_control('''\u0005'''   )   )

									self.assertFalse(_is_control('''A'''   )   )
									self.assertFalse(_is_control(''' '''   )   )
									self.assertFalse(_is_control('''\t'''   )   )
									self.assertFalse(_is_control('''\r'''   )   )

						def        __a		(	self				:      List[Any]   )							->		Dict:


									'''simple docstring'''

									self.assertTrue(_is_punctuation('''-'''   )   )
									self.assertTrue(_is_punctuation('''$'''   )   )
									self.assertTrue(_is_punctuation('''`'''   )   )
									self.assertTrue(_is_punctuation('''.'''   )   )

									self.assertFalse(_is_punctuation('''A'''   )   )
									self.assertFalse(_is_punctuation(''' '''   )   )

						def        __a		(	self				:      Optional[int]   )							->		str:


									'''simple docstring'''

									lowercase							=		self.get_tokenizer()
									lowercase							=		self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(A__   ) for t in ['''Test''', '''\xad''', '''test''']]							,							[['''[UNK]'''], [], ['''[UNK]''']]   )

									self.assertListEqual(
									    [rust_tokenizer.tokenize(A__   ) for t in ['''Test''', '''\xad''', '''test''']]							,							[['''[UNK]'''], [], ['''[UNK]''']]   )

						@slow
						def        __a		(	self				:      int   )							->		Any:


									'''simple docstring'''

									lowercase							=		self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''   )

									lowercase							=		tokenizer.encode('''sequence builders'''							,							add_special_tokens=A__   )
									lowercase							=		tokenizer.encode('''multi-sequence build'''							,							add_special_tokens=A__   )

									lowercase							=		tokenizer.build_inputs_with_special_tokens(A__   )
									lowercase							=		tokenizer.build_inputs_with_special_tokens(A__							,							A__   )

									assert encoded_sentence == [1_01] + text + [1_02]
									assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

						def        __a		(	self				:      Optional[int]   )							->		List[str]:


									'''simple docstring'''

									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
															lowercase							=		self.rust_tokenizer_class.from_pretrained(A__							,							**A__   )

															lowercase							=		f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
															lowercase							=		tokenizer_r.encode_plus(
															    A__							,							return_attention_mask=A__							,							return_token_type_ids=A__							,							return_offsets_mapping=A__							,							add_special_tokens=A__							,							)

															lowercase							=		tokenizer_r.do_lower_case if hasattr(A__							,							'''do_lower_case'''   ) else False
															lowercase							=		(
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''A'''),
															        ((1, 2), ''','''),
															        ((3, 5), '''na'''),
															        ((5, 6), '''##ï'''),
															        ((6, 8), '''##ve'''),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), '''Allen'''),
															        ((21, 23), '''##NL'''),
															        ((23, 24), '''##P'''),
															        ((25, 33), '''sentence'''),
															        ((33, 34), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''a'''),
															        ((1, 2), ''','''),
															        ((3, 8), '''naive'''),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), '''allen'''),
															        ((21, 23), '''##nl'''),
															        ((23, 24), '''##p'''),
															        ((25, 33), '''sentence'''),
															        ((33, 34), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]							,							tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
															self.assertEqual([e[0] for e in expected_results]							,							tokens['''offset_mapping''']   )



						def        __a		(	self				:      Dict   )							->		Any:


									'''simple docstring'''

									lowercase							=		['''的''', '''人''', '''有''']
									lowercase							=		''''''.join(A__   )
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
															lowercase							=		True
															lowercase							=		self.tokenizer_class.from_pretrained(A__							,							**A__   )
															lowercase							=		self.rust_tokenizer_class.from_pretrained(A__							,							**A__   )

															lowercase							=		tokenizer_p.encode(A__							,							add_special_tokens=A__   )
															lowercase							=		tokenizer_r.encode(A__							,							add_special_tokens=A__   )

															lowercase							=		tokenizer_r.convert_ids_to_tokens(A__   )
															lowercase							=		tokenizer_p.convert_ids_to_tokens(A__   )

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(A__							,							A__   )
															self.assertListEqual(A__							,							A__   )

															lowercase							=		False
															lowercase							=		self.rust_tokenizer_class.from_pretrained(A__							,							**A__   )
															lowercase							=		self.tokenizer_class.from_pretrained(A__							,							**A__   )

															lowercase							=		tokenizer_r.encode(A__							,							add_special_tokens=A__   )
															lowercase							=		tokenizer_p.encode(A__							,							add_special_tokens=A__   )

															lowercase							=		tokenizer_r.convert_ids_to_tokens(A__   )
															lowercase							=		tokenizer_p.convert_ids_to_tokens(A__   )

															# it is expected that only the first Chinese character is not preceded by "##".
															lowercase							=		[
															    f'##{token}' if idx != 0 else token for idx, token in enumerate(A__   )
															]
															self.assertListEqual(A__							,							A__   )
															self.assertListEqual(A__							,							A__   )






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class      __lowercase					(    datasets.BeamBasedBuilder    ):

						def        __a		(	self				:      Tuple   )							->		Tuple:


									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'''content''': datasets.Value('''string'''   )}   )							,							supervised_keys=__lowerCamelCase							,							)

						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Union[str, Any]							,							__lowerCamelCase				:      Tuple   )							->		int:


									'''simple docstring'''

									return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'''examples''': get_test_dummy_examples()}   )]



						def        __a		(	self				:      List[Any]							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Union[str, Any]   )							->		Optional[int]:


									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(__lowerCamelCase   )






class      __lowercase					(    datasets.BeamBasedBuilder    ):

						def        __a		(	self				:      int   )							->		Any:


									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''   )}   )}   )							,							supervised_keys=__lowerCamelCase							,							)

						def        __a		(	self				:      Optional[int]							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Optional[int]   )							->		int:


									'''simple docstring'''

									return [
									    datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'''examples''': get_test_nested_examples()}   )
									]



						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      Union[str, Any]   )							->		Tuple:


									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(__lowerCamelCase   )






def        __UpperCAmelCase    (       )->       Dict:



			"""simple docstring"""







			return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']       )]






def        __UpperCAmelCase    (       )->       List[str]:



			"""simple docstring"""







			return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']       )]






class      __lowercase					(    _A    ):

						@require_beam
						def        __a		(	self				:      Optional[int]   )							->		Optional[Any]:


									'''simple docstring'''

									lowercase							=		len(get_test_dummy_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train.arrow'   )   )   )
												self.assertDictEqual(builder.info.features							,							datasets.Features({'''content''': datasets.Value('''string'''   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												self.assertDictEqual(dset['''train'''][0]							,							get_test_dummy_examples()[0][1]   )
												self.assertDictEqual(
												    dset['''train'''][expected_num_examples - 1]							,							get_test_dummy_examples()[expected_num_examples - 1][1]   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset

						@require_beam
						def        __a		(	self				:      Union[str, Any]   )							->		List[Any]:


									'''simple docstring'''

									import apache_beam as beam

									lowercase							=		beam.io.parquetio.WriteToParquet

									lowercase							=		len(get_test_dummy_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												with patch('''apache_beam.io.parquetio.WriteToParquet'''   ) as write_parquet_mock:
															lowercase							=		partial(__lowerCamelCase							,							num_shards=2   )
															builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(
												            __lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train-00000-of-00002.arrow'   )   )   )
												self.assertTrue(
												    os.path.exists(
												        os.path.join(
												            __lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train-00000-of-00002.arrow'   )   )   )
												self.assertDictEqual(builder.info.features							,							datasets.Features({'''content''': datasets.Value('''string'''   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												# Order is not preserved when sharding, so we just check that all the elements are there
												self.assertListEqual(sorted(dset['''train''']['''content''']   )							,							sorted(['''foo''', '''bar''', '''foobar''']   )   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset

						@require_beam
						def        __a		(	self				:      int   )							->		int:


									'''simple docstring'''

									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase   )
												self.assertRaises(datasets.builder.MissingBeamOptions							,							builder.download_and_prepare   )



						@require_beam
						def        __a		(	self				:      str   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		len(get_test_nested_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		NestedBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train.arrow'   )   )   )
												self.assertDictEqual(
												    builder.info.features							,							datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''   )}   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												self.assertDictEqual(dset['''train'''][0]							,							get_test_nested_examples()[0][1]   )
												self.assertDictEqual(
												    dset['''train'''][expected_num_examples - 1]							,							get_test_nested_examples()[expected_num_examples - 1][1]   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset





