





'''simple docstring'''



import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def __init__(self     ,_lowerCamelCase=0.0_1     ,_lowerCamelCase=1000 )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							p_stop
				__lowercase										=							max_length


	def __iter__(self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							0
				__lowercase										=							False
				while not stop and count < self.max_length:
							yield count
							count += 1
							__lowercase										=							random.random() < self.p_stop






class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase=False     ,_lowerCamelCase=True )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							[
				    BatchSamplerShard(_lowerCamelCase     ,2     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )
				    for i in range(2 )
				]
				__lowercase										=							[list(_lowerCamelCase ) for batch_sampler_shard in batch_sampler_shards]
				if not split_batches:
							self.assertListEqual([len(_lowerCamelCase ) for shard in batch_sampler_shards]     ,[len(_lowerCamelCase ) for e in expected] )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				__lowercase										=							BatchSampler(range(24 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							BatchSampler(range(24 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				# Expected shouldn't change
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				# Check the shards when the dataset is a round multiple of batch size but not total batch size.
				__lowercase										=							BatchSampler(range(21 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							BatchSampler(range(21 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
				# num_processes batch.
				__lowercase										=							BatchSampler(range(22 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							BatchSampler(range(22 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
				# num_processes batch.
				__lowercase										=							BatchSampler(range(20 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							BatchSampler(range(20 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				# Check the shards when the dataset is very small.
				__lowercase										=							BatchSampler(range(2 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[[0, 1, 0]], [[1, 0, 1]]]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							BatchSampler(range(2 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[], []]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				__lowercase										=							BatchSampler(range(24 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(24 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				# Expected shouldn't change
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size.
				__lowercase										=							BatchSampler(range(22 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(22 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size or num_processes.
				__lowercase										=							BatchSampler(range(21 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(21 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )

				# Check the shards when the dataset is very small.
				__lowercase										=							BatchSampler(range(2 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[[0, 1]], [[0, 1]]]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(2 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[], []]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				__lowercase										=							BatchSampler(range(24 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(24 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				# Expected shouldn't change
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				# Check the shards when the dataset is a round multiple of batch size but not total batch size.
				__lowercase										=							BatchSampler(range(21 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(21 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
				# num_processes batch.
				__lowercase										=							BatchSampler(range(22 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(22 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
				# num_processes batch.
				__lowercase										=							BatchSampler(range(20 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(20 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
				    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				# Check the shards when the dataset is very small.
				__lowercase										=							BatchSampler(range(2 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[[0, 1]], []]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(2 )     ,batch_size=3     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[], []]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,even_batches=_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							BatchSampler(range(24 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(24 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				# Expected shouldn't change
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size.
				__lowercase										=							BatchSampler(range(22 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(22 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )

				# Check the shards when the dataset is not a round multiple of batch size or num_processes.
				__lowercase										=							BatchSampler(range(21 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(21 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[
				    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
				    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
				]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )

				# Check the shards when the dataset is very small.
				__lowercase										=							BatchSampler(range(2 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[[0, 1]], []]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )

				__lowercase										=							BatchSampler(range(2 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							[[], []]
				self.check_batch_sampler_shards(_lowerCamelCase     ,_lowerCamelCase     ,split_batches=_lowerCamelCase     ,even_batches=_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				__lowercase										=							[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
				__lowercase										=							[BatchSamplerShard(_lowerCamelCase     ,2     ,_lowerCamelCase     ,even_batches=_lowerCamelCase ) for i in range(2 )]

				self.assertEqual(len(batch_sampler_shards[0] )     ,3 )
				self.assertEqual(len(batch_sampler_shards[1] )     ,2 )

				self.assertListEqual(list(batch_sampler_shards[0] )     ,[[0, 1, 2], [5, 6, 7, 8], [12, 13]] )
				self.assertListEqual(list(batch_sampler_shards[1] )     ,[[3, 4], [9, 10, 11]] )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase=False     ,_lowerCamelCase=2     ,_lowerCamelCase=False )							->							str:







				'''simple docstring'''





				random.seed(_lowerCamelCase )
				__lowercase										=							list(_lowerCamelCase )

				__lowercase										=							[
				    IterableDatasetShard(
				        _lowerCamelCase     ,batch_size=_lowerCamelCase     ,drop_last=_lowerCamelCase     ,num_processes=_lowerCamelCase     ,process_index=_lowerCamelCase     ,split_batches=_lowerCamelCase     ,)
				    for i in range(_lowerCamelCase )
				]
				__lowercase										=							[]
				for iterable_dataset_shard in iterable_dataset_shards:
							# Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
							random.seed(_lowerCamelCase )
							iterable_dataset_lists.append(list(_lowerCamelCase ) )

				__lowercase										=							batch_size // num_processes if split_batches else batch_size
				# All iterable dataset shard should have the same length, a round multiple of shard_batch_size
				__lowercase										=							iterable_dataset_lists[0]
				for l in iterable_dataset_lists[1:]:
							self.assertEqual(len(_lowerCamelCase )     ,len(_lowerCamelCase ) )
							self.assertTrue(len(_lowerCamelCase ) % shard_batch_size == 0 )

				__lowercase										=							[]
				for idx in range(0     ,len(_lowerCamelCase )     ,_lowerCamelCase ):
							for l in iterable_dataset_lists:
										observed += l[idx : idx + shard_batch_size]

				if not drop_last:
							while len(_lowerCamelCase ) < len(_lowerCamelCase ):
										reference += reference
				self.assertListEqual(_lowerCamelCase     ,reference[: len(_lowerCamelCase )] )


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							42
				__lowercase										=							RandomIterableDataset()

				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )
				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )
				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )
				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )

				# Edge case with a very small dataset
				__lowercase										=							RandomIterableDataset(max_length=2 )

				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )
				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )
				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )
				self.check_iterable_dataset_shards(_lowerCamelCase     ,_lowerCamelCase     ,batch_size=4     ,drop_last=_lowerCamelCase     ,split_batches=_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							BatchSampler(range(16 )     ,batch_size=4     ,drop_last=_lowerCamelCase )
				__lowercase										=							SkipBatchSampler(_lowerCamelCase     ,2 )
				self.assertListEqual(list(_lowerCamelCase )     ,[[8, 9, 10, 11], [12, 13, 14, 15]] )


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							SkipDataLoader(list(range(16 ) )     ,batch_size=4     ,skip_batches=2 )
				self.assertListEqual([t.tolist() for t in dataloader]     ,[[8, 9, 10, 11], [12, 13, 14, 15]] )


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							DataLoader(list(range(16 ) )     ,batch_size=4 )
				__lowercase										=							skip_first_batches(_lowerCamelCase     ,num_batches=2 )
				self.assertListEqual([t.tolist() for t in new_dataloader]     ,[[8, 9, 10, 11], [12, 13, 14, 15]] )


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							DataLoaderShard(list(range(16 ) )     ,batch_size=4 )
				for idx, _ in enumerate(_lowerCamelCase ):
							self.assertEqual(dataloader.end_of_dataloader     ,idx == 3 )

				# Test it also works on the second iteration
				for idx, _ in enumerate(_lowerCamelCase ):
							self.assertEqual(dataloader.end_of_dataloader     ,idx == 3 )


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				Accelerator()
				__lowercase										=							DataLoaderDispatcher(range(16 )     ,batch_size=4 )
				for idx, _ in enumerate(_lowerCamelCase ):
							self.assertEqual(dataloader.end_of_dataloader     ,idx == 3 )

				# Test it also works on the second iteration
				for idx, _ in enumerate(_lowerCamelCase ):
							self.assertEqual(dataloader.end_of_dataloader     ,idx == 3 )












'''simple docstring'''



import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def __init__(self     ,_lowerCamelCase = "▁"     ,_lowerCamelCase = True     ,_lowerCamelCase = "<unk>"     ,_lowerCamelCase = "</s>"     ,_lowerCamelCase = "<pad>"     ,)							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							{
				    '''pad''': {'''id''': 0, '''token''': pad_token},
				    '''eos''': {'''id''': 1, '''token''': eos_token},
				    '''unk''': {'''id''': 2, '''token''': unk_token},
				}

				__lowercase										=							[None] * len(self.special_tokens )
				for token_dict in self.special_tokens.values():
							__lowercase										=							token_dict['''token''']

				__lowercase										=							Tokenizer(Unigram() )

				__lowercase										=							normalizers.Sequence(
				    [
				        normalizers.Nmt(),
				        normalizers.NFKC(),
				        normalizers.Replace(Regex(''' {2,}''' )     ,''' ''' ),
				        normalizers.Lowercase(),
				    ] )
				__lowercase										=							pre_tokenizers.Sequence(
				    [
				        pre_tokenizers.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase ),
				        pre_tokenizers.Digits(individual_digits=_lowerCamelCase ),
				        pre_tokenizers.Punctuation(),
				    ] )
				__lowercase										=							decoders.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase )

				__lowercase										=							TemplateProcessing(
				    single=f"$A {self.special_tokens['eos']['token']}"     ,special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]     ,)

				__lowercase										=							{
				    '''model''': '''SentencePieceUnigram''',
				    '''replacement''': replacement,
				    '''add_prefix_space''': add_prefix_space,
				}

				super().__init__(_lowerCamelCase     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				if isinstance(_lowerCamelCase     ,_lowerCamelCase ):
							__lowercase										=							[files]
				self._tokenizer.train(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							List[str]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				self._tokenizer.train_from_iterator(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							json.loads(self._tokenizer.to_str() )

				__lowercase										=							self.special_tokens['''unk''']['''id''']

				__lowercase										=							Tokenizer.from_str(json.dumps(_lowerCamelCase ) )





