



"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_			   =					logging.get_logger(__name__)

a_			   =					{
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						_A		:							Dict							       =		"""falcon"""
						_A		:							str							       =		["""past_key_values"""]







						def __init__(self   ,		lowercase__=6_50_24   ,		lowercase__=45_44   ,		lowercase__=32   ,		lowercase__=71   ,		lowercase__=1e-5   ,		lowercase__=0.02   ,		lowercase__=True   ,		lowercase__=0.0   ,		lowercase__=0.0   ,		lowercase__=None   ,		lowercase__=False   ,		lowercase__=False   ,		lowercase__=True   ,		lowercase__=True   ,		lowercase__=False   ,		lowercase__=11   ,		lowercase__=11   ,		**lowercase__   ,		):
											snake_case_					:		Optional[Any] 					=					vocab_size
											# Backward compatibility with n_embed kwarg
											snake_case_					:		str 					=					kwargs.pop("""n_embed"""   ,		lowercase__       )
											snake_case_					:		Any 					=					hidden_size if n_embed is None else n_embed
											snake_case_					:		List[str] 					=					num_hidden_layers
											snake_case_					:		Optional[int] 					=					num_attention_heads
											snake_case_					:		Optional[int] 					=					layer_norm_epsilon
											snake_case_					:		List[str] 					=					initializer_range
											snake_case_					:		Union[str, Any] 					=					use_cache
											snake_case_					:		Dict 					=					hidden_dropout
											snake_case_					:		Optional[Any] 					=					attention_dropout

											snake_case_					:		Tuple 					=					bos_token_id
											snake_case_					:		Tuple 					=					eos_token_id
											snake_case_					:		Optional[Any] 					=					num_attention_heads if num_kv_heads is None else num_kv_heads
											snake_case_					:		Union[str, Any] 					=					alibi
											snake_case_					:		Tuple 					=					new_decoder_architecture
											snake_case_					:		List[Any] 					=					multi_query  # Ignored when new_decoder_architecture is True
											snake_case_					:		Tuple 					=					parallel_attn
											snake_case_					:		Union[str, Any] 					=					bias

											super().__init__(bos_token_id=lowercase__   ,		eos_token_id=lowercase__   ,		**lowercase__       )







						@property
						def 						__UpperCamelCase							(self       ):
											return self.hidden_size // self.num_attention_heads







						@property
						def 						__UpperCamelCase							(self       ):
											return not self.alibi









"""simple docstring"""

import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
a_			   =					logging.getLogger(__name__)

if __name__ == "__main__":
			a_			   =					argparse.ArgumentParser(
			    description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
			)
			parser.add_argument(
			    '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
			)
			parser.add_argument(
			    '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
			)
			parser.add_argument('''--vocab_size''', default=30522, type=int)
			a_			   =					parser.parse_args()

			logger.info(F'''Loading data from {args.data_file}''')
			with open(args.data_file, '''rb''') as fp:
						a_			   =					pickle.load(fp)

			logger.info('''Counting occurrences for MLM.''')
			a_			   =					Counter()
			for tk_ids in data:
						counter.update(tk_ids)
			a_			   =					[0] * args.vocab_size
			for k, v in counter.items():
						a_			   =					v

			logger.info(F'''Dump to {args.token_counts_dump}''')
			with open(args.token_counts_dump, '''wb''') as handle:
						pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)




