



import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)


class   snake_case_ (lowercase__  ):




       """simple docstring"""






       def __init__(     self     ,lowercase=0.01     ,lowercase=1000):

             """simple docstring"""





             UpperCAmelCase_  :	List[Any]				      =		p_stop
             UpperCAmelCase_  :	int				      =		max_length



       def __iter__(     self):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		0
             UpperCAmelCase_  :	List[Any]				      =		False
             while not stop and count < self.max_length:
                   yield count
                   count += 1
                   UpperCAmelCase_  :	Optional[int]				      =		random.random() < self.p_stop


class   snake_case_ (unittest.TestCase  ):




       """simple docstring"""






       def  A_   (     self     ,lowercase     ,lowercase     ,lowercase=False     ,lowercase=True):

             """simple docstring"""





             UpperCAmelCase_  :	List[Any]				      =		[
                 BatchSamplerShard(lowercase     ,2     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)
                 for i in range(2)
             ]
             UpperCAmelCase_  :	Any				      =		[list(lowercase) for batch_sampler_shard in batch_sampler_shards]
             if not split_batches:
                   self.assertListEqual([len(lowercase) for shard in batch_sampler_shards]     ,[len(lowercase) for e in expected])
             self.assertListEqual(lowercase     ,lowercase)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	List[Any]				      =		BatchSampler(range(24)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Tuple				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             UpperCAmelCase_  :	int				      =		BatchSampler(range(24)     ,batch_size=3     ,drop_last=lowercase)
             # Expected shouldn't change
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             # Check the shards when the dataset is a round multiple of batch size but not total batch size.
             UpperCAmelCase_  :	int				      =		BatchSampler(range(21)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[str]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             UpperCAmelCase_  :	Tuple				      =		BatchSampler(range(21)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	int				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
             # num_processes batch.
             UpperCAmelCase_  :	Tuple				      =		BatchSampler(range(22)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[int]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             UpperCAmelCase_  :	Optional[int]				      =		BatchSampler(range(22)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[int]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
             # num_processes batch.
             UpperCAmelCase_  :	Optional[Any]				      =		BatchSampler(range(20)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[Any]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             UpperCAmelCase_  :	Tuple				      =		BatchSampler(range(20)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Union[str, Any]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             # Check the shards when the dataset is very small.
             UpperCAmelCase_  :	str				      =		BatchSampler(range(2)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Dict				      =		[[[0, 1, 0]], [[1, 0, 1]]]
             self.check_batch_sampler_shards(lowercase     ,lowercase)

             UpperCAmelCase_  :	Union[str, Any]				      =		BatchSampler(range(2)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[int]				      =		[[], []]
             self.check_batch_sampler_shards(lowercase     ,lowercase)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Optional[Any]				      =		BatchSampler(range(24)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[Any]				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)

             UpperCAmelCase_  :	List[Any]				      =		BatchSampler(range(24)     ,batch_size=4     ,drop_last=lowercase)
             # Expected shouldn't change
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)

             # Check the shards when the dataset is not a round multiple of batch size.
             UpperCAmelCase_  :	int				      =		BatchSampler(range(22)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[str]				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)

             UpperCAmelCase_  :	List[Any]				      =		BatchSampler(range(22)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	int				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)

             # Check the shards when the dataset is not a round multiple of batch size or num_processes.
             UpperCAmelCase_  :	List[str]				      =		BatchSampler(range(21)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	str				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)

             UpperCAmelCase_  :	str				      =		BatchSampler(range(21)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	Dict				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)

             # Check the shards when the dataset is very small.
             UpperCAmelCase_  :	Dict				      =		BatchSampler(range(2)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[str]				      =		[[[0, 1]], [[0, 1]]]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)

             UpperCAmelCase_  :	Optional[Any]				      =		BatchSampler(range(2)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[int]				      =		[[], []]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	str				      =		BatchSampler(range(24)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	int				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Any				      =		BatchSampler(range(24)     ,batch_size=3     ,drop_last=lowercase)
             # Expected shouldn't change
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             # Check the shards when the dataset is a round multiple of batch size but not total batch size.
             UpperCAmelCase_  :	List[str]				      =		BatchSampler(range(21)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	int				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Optional[Any]				      =		BatchSampler(range(21)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[int]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
             # num_processes batch.
             UpperCAmelCase_  :	Any				      =		BatchSampler(range(22)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Dict				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Optional[int]				      =		BatchSampler(range(22)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[int]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
             # num_processes batch.
             UpperCAmelCase_  :	Optional[Any]				      =		BatchSampler(range(20)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[Any]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Dict				      =		BatchSampler(range(20)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[str]				      =		[
                 [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
                 [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             # Check the shards when the dataset is very small.
             UpperCAmelCase_  :	int				      =		BatchSampler(range(2)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[Any]				      =		[[[0, 1]], []]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Any				      =		BatchSampler(range(2)     ,batch_size=3     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[str]				      =		[[], []]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,even_batches=lowercase)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	str				      =		BatchSampler(range(24)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[int]				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Tuple				      =		BatchSampler(range(24)     ,batch_size=4     ,drop_last=lowercase)
             # Expected shouldn't change
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)

             # Check the shards when the dataset is not a round multiple of batch size.
             UpperCAmelCase_  :	Any				      =		BatchSampler(range(22)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	str				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Any				      =		BatchSampler(range(22)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[Any]				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)

             # Check the shards when the dataset is not a round multiple of batch size or num_processes.
             UpperCAmelCase_  :	str				      =		BatchSampler(range(21)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[str]				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	int				      =		BatchSampler(range(21)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[Any]				      =		[
                 [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
                 [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
             ]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)

             # Check the shards when the dataset is very small.
             UpperCAmelCase_  :	List[str]				      =		BatchSampler(range(2)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	List[str]				      =		[[[0, 1]], []]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)

             UpperCAmelCase_  :	Tuple				      =		BatchSampler(range(2)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	Union[str, Any]				      =		[[], []]
             self.check_batch_sampler_shards(lowercase     ,lowercase     ,split_batches=lowercase     ,even_batches=lowercase)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
             UpperCAmelCase_  :	Optional[Any]				      =		[BatchSamplerShard(lowercase     ,2     ,lowercase     ,even_batches=lowercase) for i in range(2)]

             self.assertEqual(len(batch_sampler_shards[0])     ,3)
             self.assertEqual(len(batch_sampler_shards[1])     ,2)

             self.assertListEqual(list(batch_sampler_shards[0])     ,[[0, 1, 2], [5, 6, 7, 8], [12, 13]])
             self.assertListEqual(list(batch_sampler_shards[1])     ,[[3, 4], [9, 10, 11]])



       def  A_   (     self     ,lowercase     ,lowercase     ,lowercase     ,lowercase=False     ,lowercase=2     ,lowercase=False):

             """simple docstring"""





             random.seed(lowercase)
             UpperCAmelCase_  :	str				      =		list(lowercase)

             UpperCAmelCase_  :	Optional[Any]				      =		[
                 IterableDatasetShard(
                     lowercase     ,batch_size=lowercase     ,drop_last=lowercase     ,num_processes=lowercase     ,process_index=lowercase     ,split_batches=lowercase     ,)
                 for i in range(lowercase)
             ]
             UpperCAmelCase_  :	List[str]				      =		[]
             for iterable_dataset_shard in iterable_dataset_shards:
                   # Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
                   random.seed(lowercase)
                   iterable_dataset_lists.append(list(lowercase))

             UpperCAmelCase_  :	Dict				      =		batch_size // num_processes if split_batches else batch_size
             # All iterable dataset shard should have the same length, a round multiple of shard_batch_size
             UpperCAmelCase_  :	Optional[Any]				      =		iterable_dataset_lists[0]
             for l in iterable_dataset_lists[1:]:
                   self.assertEqual(len(lowercase)     ,len(lowercase))
                   self.assertTrue(len(lowercase) % shard_batch_size == 0)

             UpperCAmelCase_  :	Tuple				      =		[]
             for idx in range(0     ,len(lowercase)     ,lowercase):
                   for l in iterable_dataset_lists:
                         observed += l[idx : idx + shard_batch_size]

             if not drop_last:
                   while len(lowercase) < len(lowercase):
                         reference += reference
             self.assertListEqual(lowercase     ,reference[: len(lowercase)])



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Any				      =		42
             UpperCAmelCase_  :	Any				      =		RandomIterableDataset()

             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)
             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)
             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)
             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)

             # Edge case with a very small dataset
             UpperCAmelCase_  :	Tuple				      =		RandomIterableDataset(max_length=2)

             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)
             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)
             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)
             self.check_iterable_dataset_shards(lowercase     ,lowercase     ,batch_size=4     ,drop_last=lowercase     ,split_batches=lowercase)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		BatchSampler(range(16)     ,batch_size=4     ,drop_last=lowercase)
             UpperCAmelCase_  :	Optional[Any]				      =		SkipBatchSampler(lowercase     ,2)
             self.assertListEqual(list(lowercase)     ,[[8, 9, 10, 11], [12, 13, 14, 15]])



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	List[Any]				      =		SkipDataLoader(list(range(16))     ,batch_size=4     ,skip_batches=2)
             self.assertListEqual([t.tolist() for t in dataloader]     ,[[8, 9, 10, 11], [12, 13, 14, 15]])



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	str				      =		DataLoader(list(range(16))     ,batch_size=4)
             UpperCAmelCase_  :	Optional[int]				      =		skip_first_batches(lowercase     ,num_batches=2)
             self.assertListEqual([t.tolist() for t in new_dataloader]     ,[[8, 9, 10, 11], [12, 13, 14, 15]])



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Tuple				      =		DataLoaderShard(list(range(16))     ,batch_size=4)
             for idx, _ in enumerate(lowercase):
                   self.assertEqual(dataloader.end_of_dataloader     ,idx == 3)

             # Test it also works on the second iteration
             for idx, _ in enumerate(lowercase):
                   self.assertEqual(dataloader.end_of_dataloader     ,idx == 3)



       def  A_   (     self):

             """simple docstring"""





             Accelerator()
             UpperCAmelCase_  :	Dict				      =		DataLoaderDispatcher(range(16)     ,batch_size=4)
             for idx, _ in enumerate(lowercase):
                   self.assertEqual(dataloader.end_of_dataloader     ,idx == 3)

             # Test it also works on the second iteration
             for idx, _ in enumerate(lowercase):
                   self.assertEqual(dataloader.end_of_dataloader     ,idx == 3)





import logging

from transformers import PretrainedConfig


__lowerCamelCase         =			logging.getLogger(__name__)


__lowerCamelCase         =			{
    '''bertabs-finetuned-cnndm''': '''https://huggingface.co/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization/resolve/main/config.json''',
}


class   snake_case_ (lowercase__  ):




       """simple docstring"""






       _lowerCamelCase			    =				"""bertabs"""



       def __init__(     self     ,lowercase=30522     ,lowercase=512     ,lowercase=6     ,lowercase=512     ,lowercase=8     ,lowercase=512     ,lowercase=0.2     ,lowercase=6     ,lowercase=768     ,lowercase=8     ,lowercase=2048     ,lowercase=0.2     ,**lowercase     ,):

             """simple docstring"""





             super().__init__(**lowercase)

             UpperCAmelCase_  :	Any				      =		vocab_size
             UpperCAmelCase_  :	int				      =		max_pos

             UpperCAmelCase_  :	List[str]				      =		enc_layers
             UpperCAmelCase_  :	List[str]				      =		enc_hidden_size
             UpperCAmelCase_  :	Any				      =		enc_heads
             UpperCAmelCase_  :	Any				      =		enc_ff_size
             UpperCAmelCase_  :	Optional[int]				      =		enc_dropout

             UpperCAmelCase_  :	List[str]				      =		dec_layers
             UpperCAmelCase_  :	List[Any]				      =		dec_hidden_size
             UpperCAmelCase_  :	List[str]				      =		dec_heads
             UpperCAmelCase_  :	List[Any]				      =		dec_ff_size
             UpperCAmelCase_  :	List[str]				      =		dec_dropout
