





import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
_a						=   logging.getLogger(__name__)





def lowerCAmelCase__()       ->   Tuple:







	'''simple docstring'''







	lowerCamelCase__              =	argparse.ArgumentParser(
	    description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''							)
	parser.add_argument('''--file_path''' ,type=__snake_case ,default='''data/dump.txt''' ,help='''The path to the data.'''							)
	parser.add_argument('''--tokenizer_type''' ,type=__snake_case ,default='''bert''' ,choices=['''bert''', '''roberta''', '''gpt2''']							)
	parser.add_argument('''--tokenizer_name''' ,type=__snake_case ,default='''bert-base-uncased''' ,help='''The tokenizer to use.'''							)
	parser.add_argument('''--dump_file''' ,type=__snake_case ,default='''data/dump''' ,help='''The dump file prefix.'''							)
	lowerCamelCase__              =	parser.parse_args()

	logger.info(F'Loading Tokenizer ({args.tokenizer_name})'							)
	if args.tokenizer_type == "bert":
		lowerCamelCase__              =	BertTokenizer.from_pretrained(args.tokenizer_name							)
		lowerCamelCase__              =	tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
		lowerCamelCase__              =	tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
	elif args.tokenizer_type == "roberta":
		lowerCamelCase__              =	RobertaTokenizer.from_pretrained(args.tokenizer_name							)
		lowerCamelCase__              =	tokenizer.special_tokens_map['''cls_token''']  # `<s>`
		lowerCamelCase__              =	tokenizer.special_tokens_map['''sep_token''']  # `</s>`
	elif args.tokenizer_type == "gpt2":
		lowerCamelCase__              =	GPTaTokenizer.from_pretrained(args.tokenizer_name							)
		lowerCamelCase__              =	tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
		lowerCamelCase__              =	tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

	logger.info(F'Loading text from {args.file_path}'							)
	with open(args.file_path ,'''r''' ,encoding='''utf8'''							) as fp:
		lowerCamelCase__              =	fp.readlines()

	logger.info('''Start encoding'''							)
	logger.info(F'{len(__snake_case							)} examples to process.'							)

	lowerCamelCase__              =	[]
	lowerCamelCase__              =	0
	lowerCamelCase__              =	10000
	lowerCamelCase__              =	time.time()
	for text in data:
		lowerCamelCase__              =	F'{bos} {text.strip()} {sep}'
		lowerCamelCase__              =	tokenizer.encode(__snake_case ,add_special_tokens=__snake_case							)
		rslt.append(__snake_case							)

		iter += 1
		if iter % interval == 0:
			lowerCamelCase__              =	time.time()
			logger.info(F'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'							)
			lowerCamelCase__              =	time.time()
	logger.info('''Finished binarization'''							)
	logger.info(F'{len(__snake_case							)} examples processed.'							)

	lowerCamelCase__              =	F'{args.dump_file}.{args.tokenizer_name}.pickle'
	lowerCamelCase__              =	tokenizer.vocab_size
	if vocab_size < (1 << 16):
		lowerCamelCase__              =	[np.uintaa(__snake_case							) for d in rslt]
	else:
		lowerCamelCase__              =	[np.intaa(__snake_case							) for d in rslt]
	random.shuffle(rslt_							)
	logger.info(F'Dump to {dp_file}'							)
	with open(__snake_case ,'''wb'''							) as handle:
		pickle.dump(rslt_ ,__snake_case ,protocol=pickle.HIGHEST_PROTOCOL							)


if __name__ == "__main__":
			main()










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_a						=   {
    "configuration_funnel": ["FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP", "FunnelConfig"],
    "convert_funnel_original_tf_checkpoint_to_pytorch": [],
    "tokenization_funnel": ["FunnelTokenizer"],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   ["FunnelTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   [
			    "FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "FunnelBaseModel",
			    "FunnelForMaskedLM",
			    "FunnelForMultipleChoice",
			    "FunnelForPreTraining",
			    "FunnelForQuestionAnswering",
			    "FunnelForSequenceClassification",
			    "FunnelForTokenClassification",
			    "FunnelModel",
			    "FunnelPreTrainedModel",
			    "load_tf_weights_in_funnel",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   [
			    "TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFFunnelBaseModel",
			    "TFFunnelForMaskedLM",
			    "TFFunnelForMultipleChoice",
			    "TFFunnelForPreTraining",
			    "TFFunnelForQuestionAnswering",
			    "TFFunnelForSequenceClassification",
			    "TFFunnelForTokenClassification",
			    "TFFunnelModel",
			    "TFFunnelPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
			from .tokenization_funnel import FunnelTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_funnel_fast import FunnelTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_funnel import (
						    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    FunnelBaseModel,
						    FunnelForMaskedLM,
						    FunnelForMultipleChoice,
						    FunnelForPreTraining,
						    FunnelForQuestionAnswering,
						    FunnelForSequenceClassification,
						    FunnelForTokenClassification,
						    FunnelModel,
						    FunnelPreTrainedModel,
						    load_tf_weights_in_funnel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_funnel import (
						    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFFunnelBaseModel,
						    TFFunnelForMaskedLM,
						    TFFunnelForMultipleChoice,
						    TFFunnelForPreTraining,
						    TFFunnelForQuestionAnswering,
						    TFFunnelForSequenceClassification,
						    TFFunnelForTokenClassification,
						    TFFunnelModel,
						    TFFunnelPreTrainedModel,
						)

else:
			import sys

			_a						=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



