




"""simple docstring"""






import argparse
import gc
import json
import os
import re

import torch
from huggingface_hub import hf_hub_download

from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizerFast, RwkvConfig
from transformers.modeling_utils import WEIGHTS_INDEX_NAME, shard_checkpoint


lowercase__  		=		{
    """169M""": 12,
    """430M""": 24,
    """1B5""": 24,
    """3B""": 32,
    """7B""": 32,
    """14B""": 40,
}

lowercase__  		=		{
    """169M""": 768,
    """430M""": 1024,
    """1B5""": 2048,
    """3B""": 2560,
    """7B""": 4096,
    """14B""": 5120,
}







def   _snake_case    (       lowercase__	):
				_lowerCamelCase  :    str					=					list(state_dict.keys()	)
				for name in state_dict_keys:
								_lowerCamelCase  :    Union[str, Any]					=					state_dict.pop(lowercase__	)
								# emb -> embedding
								if name.startswith('emb.'	):
												_lowerCamelCase  :    List[Any]					=					name.replace('emb.'							,					'embeddings.'	)
								# ln_0 -> pre_ln (only present at block 0)
								if name.startswith('blocks.0.ln0'	):
												_lowerCamelCase  :    str					=					name.replace('blocks.0.ln0'							,					'blocks.0.pre_ln'	)
								# att -> attention
								_lowerCamelCase  :    List[Any]					=					re.sub(r'blocks\.(\d+)\.att'							,					r'blocks.\1.attention'							,					lowercase__	)
								# ffn -> feed_forward
								_lowerCamelCase  :    Any					=					re.sub(r'blocks\.(\d+)\.ffn'							,					r'blocks.\1.feed_forward'							,					lowercase__	)
								# time_mix_k -> time_mix_key and reshape
								if name.endswith('.time_mix_k'	):
												_lowerCamelCase  :    Dict					=					name.replace('.time_mix_k'							,					'.time_mix_key'	)
								# time_mix_v -> time_mix_value and reshape
								if name.endswith('.time_mix_v'	):
												_lowerCamelCase  :    int					=					name.replace('.time_mix_v'							,					'.time_mix_value'	)
								# time_mix_r -> time_mix_key and reshape
								if name.endswith('.time_mix_r'	):
												_lowerCamelCase  :    List[str]					=					name.replace('.time_mix_r'							,					'.time_mix_receptance'	)

								if name != "head.weight":
												_lowerCamelCase  :    List[str]					=					'rwkv.' + name

								_lowerCamelCase  :    Tuple					=					weight
				return state_dict







def   _snake_case    (       lowercase__							,					lowercase__							,					lowercase__							,					lowercase__=None							,					lowercase__=None							,					lowercase__=False							,					lowercase__=None	):
				# 1. If possible, build the tokenizer.
				if tokenizer_file is None:
								print('No `--tokenizer_file` provided, we will use the default tokenizer.'	)
								_lowerCamelCase  :    Any					=					50277
								_lowerCamelCase  :    str					=					AutoTokenizer.from_pretrained('EleutherAI/gpt-neox-20b'	)
				else:
								_lowerCamelCase  :    str					=					PreTrainedTokenizerFast(tokenizer_file=lowercase__	)
								_lowerCamelCase  :    List[Any]					=					len(lowercase__	)
				tokenizer.save_pretrained(lowercase__	)

				# 2. Build the config
				_lowerCamelCase  :    str					=					list(NUM_HIDDEN_LAYERS_MAPPING.keys()	)
				if size is None:
								# Try to infer size from the checkpoint name
								for candidate in possible_sizes:
												if candidate in checkpoint_file:
																_lowerCamelCase  :    Optional[int]					=					candidate
																break
								if size is None:
												raise ValueError('Could not infer the size, please provide it with the `--size` argument.'	)
				if size not in possible_sizes:
								raise ValueError(f'''`size` should be one of {possible_sizes}, got {size}.'''	)

				_lowerCamelCase  :    Optional[Any]					=					RwkvConfig(
				    vocab_size=lowercase__							,					num_hidden_layers=NUM_HIDDEN_LAYERS_MAPPING[size]							,					hidden_size=HIDEN_SIZE_MAPPING[size]							,					)
				config.save_pretrained(lowercase__	)

				# 3. Download model file then convert state_dict
				_lowerCamelCase  :    str					=					hf_hub_download(lowercase__							,					lowercase__	)
				_lowerCamelCase  :    str					=					torch.load(lowercase__							,					map_location='cpu'	)
				_lowerCamelCase  :    Any					=					convert_state_dict(lowercase__	)

				# 4. Split in shards and save
				_lowerCamelCase,				_lowerCamelCase  :    Optional[Any]					=					shard_checkpoint(lowercase__	)
				for shard_file, shard in shards.items():
								torch.save(lowercase__							,					os.path.join(lowercase__							,					lowercase__	)	)

				if index is not None:
								_lowerCamelCase  :    Optional[Any]					=					os.path.join(lowercase__							,					lowercase__	)
								# Save the index as well
								with open(lowercase__							,					'w'							,					encoding='utf-8'	) as f:
												_lowerCamelCase  :    int					=					json.dumps(lowercase__							,					indent=2							,					sort_keys=lowercase__	) + '\n'
												f.write(lowercase__	)

								# 5. Clean up shards (for some reason the file PyTorch saves take the same space as the whole state_dict
								print(
								    'Cleaning up shards. This may error with an OOM error, it this is the case don\'t worry you still have converted the model.'	)
								_lowerCamelCase  :    List[str]					=					list(shards.keys()	)

								del state_dict
								del shards
								gc.collect()

								for shard_file in shard_files:
												_lowerCamelCase  :    List[str]					=					torch.load(os.path.join(lowercase__							,					lowercase__	)	)
												torch.save({k: v.cpu().clone() for k, v in state_dict.items()}							,					os.path.join(lowercase__							,					lowercase__	)	)

				del state_dict
				gc.collect()

				if push_to_hub:
								if model_name is None:
												raise ValueError('Please provide a `model_name` to push the model to the Hub.'	)
								_lowerCamelCase  :    Any					=					AutoModelForCausalLM.from_pretrained(lowercase__	)
								model.push_to_hub(lowercase__							,					max_shard_size='2GB'	)
								tokenizer.push_to_hub(lowercase__	)


if __name__ == "__main__":
			lowercase__  		=		argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    """--repo_id""", default=None, type=str, required=True, help="""Repo ID from which to pull the checkpoint."""
			)
			parser.add_argument(
			    """--checkpoint_file""", default=None, type=str, required=True, help="""Name of the checkpoint file in the repo."""
			)
			parser.add_argument(
			    """--output_dir""", default=None, type=str, required=True, help="""Where to save the converted model."""
			)
			parser.add_argument(
			    """--tokenizer_file""",
			    default=None,
			    type=str,
			    help="""Path to the tokenizer file to use (if not provided, only the model is converted).""",
			)
			parser.add_argument(
			    """--size""",
			    default=None,
			    type=str,
			    help="""Size of the model. Will be inferred from the `checkpoint_file` if not passed.""",
			)
			parser.add_argument(
			    """--push_to_hub""",
			    action="""store_true""",
			    help="""Push to the Hub the converted model.""",
			)
			parser.add_argument(
			    """--model_name""",
			    default=None,
			    type=str,
			    help="""Name of the pushed model on the Hub, including the username / organization.""",
			)

			lowercase__  		=		parser.parse_args()
			convert_rmkv_checkpoint_to_hf_format(
			    args.repo_id,
			    args.checkpoint_file,
			    args.output_dir,
			    size=args.size,
			    tokenizer_file=args.tokenizer_file,
			    push_to_hub=args.push_to_hub,
			    model_name=args.model_name,
			)





"""simple docstring"""






def   _snake_case    (       lowercase__	):
				_lowerCamelCase  :    List[str]					=					abs(lowercase__	)
				_lowerCamelCase  :    Optional[int]					=					0
				while n > 0:
								res += n % 10
								n //= 10
				return res







def   _snake_case    (       lowercase__	):
				_lowerCamelCase  :    List[str]					=					abs(lowercase__	)
				return n if n < 10 else n % 10 + sum_of_digits(n // 10	)







def   _snake_case    (       lowercase__	):
				return sum(int(lowercase__	) for c in str(abs(lowercase__	)	)	)







def   _snake_case    (       ):
				from collections.abc import Callable
				from timeit import timeit

				def benchmark_a_function(lowercase__							,					lowercase__	) -> None:
								_lowerCamelCase  :    int					=					f'''{func.__name__}({value})'''
								_lowerCamelCase  :    Optional[Any]					=					timeit(f'''__main__.{call}'''							,					setup='import __main__'	)
								print(f'''{call:56} = {func(lowercase__	)} -- {timing:.4f} seconds'''	)

				for value in (262144, 1125899906842624, 1267650600228229401496703205376):
								for func in (sum_of_digits, sum_of_digits_recursion, sum_of_digits_compact):
												benchmark_a_function(lowercase__							,					lowercase__	)
								print()


if __name__ == "__main__":
			import doctest

			doctest.testmod()
			benchmark()