



"""simple docstring"""





import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
lowercase  							=					logging.getLogger(__name__)




def 	UpperCAmelCase     (		):

		'''simple docstring'''


		_UpperCAmelCase							 =		argparse.ArgumentParser(
		    description='Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'       )
		parser.add_argument('--file_path'     ,				type=A     ,				default='data/dump.txt'     ,				help='The path to the data.'       )
		parser.add_argument('--tokenizer_type'     ,				type=A     ,				default='bert'     ,				choices=['bert', 'roberta', 'gpt2']       )
		parser.add_argument('--tokenizer_name'     ,				type=A     ,				default='bert-base-uncased'     ,				help='The tokenizer to use.'       )
		parser.add_argument('--dump_file'     ,				type=A     ,				default='data/dump'     ,				help='The dump file prefix.'       )
		_UpperCAmelCase							 =		parser.parse_args()

		logger.info(f'Loading Tokenizer ({args.tokenizer_name})'       )
		if args.tokenizer_type == "bert":
				_UpperCAmelCase							 =		BertTokenizer.from_pretrained(args.tokenizer_name       )
				_UpperCAmelCase							 =		tokenizer.special_tokens_map['cls_token']  # `[CLS]`
				_UpperCAmelCase							 =		tokenizer.special_tokens_map['sep_token']  # `[SEP]`
		elif args.tokenizer_type == "roberta":
				_UpperCAmelCase							 =		RobertaTokenizer.from_pretrained(args.tokenizer_name       )
				_UpperCAmelCase							 =		tokenizer.special_tokens_map['cls_token']  # `<s>`
				_UpperCAmelCase							 =		tokenizer.special_tokens_map['sep_token']  # `</s>`
		elif args.tokenizer_type == "gpt2":
				_UpperCAmelCase							 =		GPTaTokenizer.from_pretrained(args.tokenizer_name       )
				_UpperCAmelCase							 =		tokenizer.special_tokens_map['bos_token']  # `<|endoftext|>`
				_UpperCAmelCase							 =		tokenizer.special_tokens_map['eos_token']  # `<|endoftext|>`

		logger.info(f'Loading text from {args.file_path}'       )
		with open(args.file_path     ,				'r'     ,				encoding='utf8'       ) as fp:
				_UpperCAmelCase							 =		fp.readlines()

		logger.info('Start encoding'       )
		logger.info(f'{len(A       )} examples to process.'       )

		_UpperCAmelCase							 =		[]
		_UpperCAmelCase							 =		0
		_UpperCAmelCase							 =		1_0000
		_UpperCAmelCase							 =		time.time()
		for text in data:
				_UpperCAmelCase							 =		f'{bos} {text.strip()} {sep}'
				_UpperCAmelCase							 =		tokenizer.encode(A     ,				add_special_tokens=A       )
				rslt.append(A       )

				iter += 1
				if iter % interval == 0:
						_UpperCAmelCase							 =		time.time()
						logger.info(f'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'       )
						_UpperCAmelCase							 =		time.time()
		logger.info('Finished binarization'       )
		logger.info(f'{len(A       )} examples processed.'       )

		_UpperCAmelCase							 =		f'{args.dump_file}.{args.tokenizer_name}.pickle'
		_UpperCAmelCase							 =		tokenizer.vocab_size
		if vocab_size < (1 << 16):
				_UpperCAmelCase							 =		[np.uintaa(A       ) for d in rslt]
		else:
				_UpperCAmelCase							 =		[np.intaa(A       ) for d in rslt]
		random.shuffle(rslt_       )
		logger.info(f'Dump to {dp_file}'       )
		with open(A     ,				'wb'       ) as handle:
				pickle.dump(rslt_     ,				A     ,				protocol=pickle.HIGHEST_PROTOCOL       )


if __name__ == "__main__":
					main()







"""simple docstring"""





from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


lowercase  							=					logging.get_logger(__name__)


lowercase  							=					TypeVar('''DatasetType''', Dataset, IterableDataset)




def 	UpperCAmelCase     (		A	:						List[DatasetType]     ,				A	:						Optional[List[float]] = None     ,				A	:						Optional[int] = None     ,				A	:						Optional[DatasetInfo] = None     ,				A	:						Optional[NamedSplit] = None     ,				A	:						Literal["first_exhausted", "all_exhausted"] = "first_exhausted"     ,				):

		'''simple docstring'''


		from .arrow_dataset import Dataset
		from .iterable_dataset import IterableDataset

		if not datasets:
				raise ValueError('Unable to interleave an empty list of datasets.'       )
		for i, dataset in enumerate(A       ):
				if not isinstance(A     ,				(Dataset, IterableDataset)       ):
						if isinstance(A     ,				(DatasetDict, IterableDatasetDict)       ):
								if not dataset:
										raise ValueError(
										    f'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
										    'is an empty dataset dictionary.'       )
								raise ValueError(
								    f'Dataset at position {i} has at least one split: {list(A       )}\n'
								    f'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(A       )       )}\']'       )
						raise ValueError(
						    f'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(A       ).__name__}.'       )
				if i == 0:
						_UpperCAmelCase ,					_UpperCAmelCase							 =		(
						    (Dataset, IterableDataset) if isinstance(A     ,				A       ) else (IterableDataset, Dataset)
						)
				elif not isinstance(A     ,				A       ):
						raise ValueError(
						    f'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'       )
		if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
				raise ValueError(f'{stopping_strategy} is not supported. Please enter a valid stopping_strategy.'       )
		if dataset_type is Dataset:
				return _interleave_map_style_datasets(
				    A     ,				A     ,				A     ,				info=A     ,				split=A     ,				stopping_strategy=A       )
		else:
				return _interleave_iterable_datasets(
				    A     ,				A     ,				A     ,				info=A     ,				split=A     ,				stopping_strategy=A       )






def 	UpperCAmelCase     (		A	:						List[DatasetType]     ,				A	:						Optional[DatasetInfo] = None     ,				A	:						Optional[NamedSplit] = None     ,				A	:						int = 0     ,				):

		'''simple docstring'''


		if not dsets:
				raise ValueError('Unable to concatenate an empty list of datasets.'       )
		for i, dataset in enumerate(A       ):
				if not isinstance(A     ,				(Dataset, IterableDataset)       ):
						if isinstance(A     ,				(DatasetDict, IterableDatasetDict)       ):
								if not dataset:
										raise ValueError(
										    f'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
										    'is an empty dataset dictionary.'       )
								raise ValueError(
								    f'Dataset at position {i} has at least one split: {list(A       )}\n'
								    f'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(A       )       )}\']'       )
						raise ValueError(
						    f'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(A       ).__name__}.'       )
				if i == 0:
						_UpperCAmelCase ,					_UpperCAmelCase							 =		(
						    (Dataset, IterableDataset) if isinstance(A     ,				A       ) else (IterableDataset, Dataset)
						)
				elif not isinstance(A     ,				A       ):
						raise ValueError(
						    f'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'       )
		if dataset_type is Dataset:
				return _concatenate_map_style_datasets(A     ,				info=A     ,				split=A     ,				axis=A       )
		else:
				return _concatenate_iterable_datasets(A     ,				info=A     ,				split=A     ,				axis=A       )


