






"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tensorflow_text_available, is_torch_available


lowercase  							=					{
    '''configuration_ernie''': ['''ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ErnieConfig''', '''ErnieOnnxConfig'''],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase  							=					[
					    '''ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST''',
					    '''ErnieForCausalLM''',
					    '''ErnieForMaskedLM''',
					    '''ErnieForMultipleChoice''',
					    '''ErnieForNextSentencePrediction''',
					    '''ErnieForPreTraining''',
					    '''ErnieForQuestionAnswering''',
					    '''ErnieForSequenceClassification''',
					    '''ErnieForTokenClassification''',
					    '''ErnieModel''',
					    '''ErniePreTrainedModel''',
					]

if TYPE_CHECKING:
					from .configuration_ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig, ErnieOnnxConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_ernie import (
										    ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST,
										    ErnieForCausalLM,
										    ErnieForMaskedLM,
										    ErnieForMultipleChoice,
										    ErnieForNextSentencePrediction,
										    ErnieForPreTraining,
										    ErnieForQuestionAnswering,
										    ErnieForSequenceClassification,
										    ErnieForTokenClassification,
										    ErnieModel,
										    ErniePreTrainedModel,
										)

else:
					import sys

					lowercase  							=					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)







"""simple docstring"""





import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
lowercase  							=					logging.getLogger(__name__)

if __name__ == "__main__":
					lowercase  							=					argparse.ArgumentParser(
					    description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
					)
					parser.add_argument(
					    '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
					)
					parser.add_argument(
					    '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
					)
					parser.add_argument('''--vocab_size''', default=3_05_22, type=int)
					lowercase  							=					parser.parse_args()

					logger.info(F'''Loading data from {args.data_file}''')
					with open(args.data_file, '''rb''') as fp:
										lowercase  							=					pickle.load(fp)

					logger.info('''Counting occurrences for MLM.''')
					lowercase  							=					Counter()
					for tk_ids in data:
										counter.update(tk_ids)
					lowercase  							=					[0] * args.vocab_size
					for k, v in counter.items():
										lowercase  							=					v

					logger.info(F'''Dump to {args.token_counts_dump}''')
					with open(args.token_counts_dump, '''wb''') as handle:
										pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)


