




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
     from transformers import LlamaTokenizerFast
except ImportError as e:
     warnings.warn(e)
     warnings.warn(
         "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
     )
     _UpperCAmelCase	:						Any			=     None
_UpperCAmelCase	:						List[Any]			=     {
    "7B": 11008,
    "13B": 13824,
    "30B": 17920,
    "65B": 22016,
    "70B": 28672,
}
_UpperCAmelCase	:						int			=     {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase=1,			lowerCamelCase=256   ):
  return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)



def       UpperCAmelCase__  (     lowerCamelCase   ):


  with open(lowerCAmelCase_,			"r"   ) as f:
    return json.load(lowerCAmelCase_   )



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):


  with open(lowerCAmelCase_,			"w"   ) as f:
    json.dump(lowerCAmelCase_,			lowerCAmelCase_   )



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase=True   ):
  os.makedirs(lowerCAmelCase_,			exist_ok=lowerCAmelCase_   )
  lowercase    :Optional[int]      	=						os.path.join(lowerCAmelCase_,			"tmp"   )
  os.makedirs(lowerCAmelCase_,			exist_ok=lowerCAmelCase_   )

  lowercase    :Tuple      	=						read_json(os.path.join(lowerCAmelCase_,			"params.json"   )   )
  lowercase    :str      	=						NUM_SHARDS[model_size]
  lowercase    :Dict      	=						params["n_layers"]
  lowercase    :List[str]      	=						params["n_heads"]
  lowercase    :Dict      	=						n_heads // num_shards
  lowercase    :List[str]      	=						params["dim"]
  lowercase    :str      	=						dim // n_heads
  lowercase    :Optional[int]      	=						10000.0
  lowercase    :str      	=						1.0 / (base ** (torch.arange(0,			lowerCAmelCase_,			2   ).float() / dims_per_head))

  if "n_kv_heads" in params:
    lowercase    :int      	=						params["n_kv_heads"]  # for GQA / MQA
    lowercase    :Dict      	=						n_heads_per_shard // num_key_value_heads
    lowercase    :Tuple      	=						dim // num_key_value_heads
  else:  # compatibility with other checkpoints
    lowercase    :List[str]      	=						n_heads
    lowercase    :Optional[int]      	=						n_heads_per_shard
    lowercase    :List[Any]      	=						dim

  # permute for sliced rotary
  def permute(lowerCamelCase,			lowerCamelCase=n_heads,			lowerCamelCase=dim,			lowerCamelCase=dim   ):
    return w.view(lowerCAmelCase_,			dima // n_heads // 2,			2,			lowerCAmelCase_   ).transpose(1,			2   ).reshape(lowerCAmelCase_,			lowerCAmelCase_   )

  print(F"Fetching all parameters from the checkpoint at {input_base_path}."   )
  # Load weights
  if model_size == "7B":
    # Not sharded
    # (The sharded implementation would also work, but this is simpler.)
    lowercase    :str      	=						torch.load(os.path.join(lowerCAmelCase_,			"consolidated.00.pth"   ),			map_location="cpu"   )
  else:
    # Sharded
    lowercase    :Tuple      	=						[
        torch.load(os.path.join(lowerCAmelCase_,			F"consolidated.{i:02d}.pth"   ),			map_location="cpu"   )
        for i in range(lowerCAmelCase_   )
    ]
  lowercase    :Any      	=						0
  lowercase    :Union[str, Any]      	=						{"weight_map": {}}
  for layer_i in range(lowerCAmelCase_   ):
    lowercase    :str      	=						F"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
      # Unsharded
      lowercase    :Optional[Any]      	=						{
          F"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
              loaded[F"layers.{layer_i}.attention.wq.weight"]   ),
          F"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
              loaded[F"layers.{layer_i}.attention.wk.weight"]   ),
          F"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[F"layers.{layer_i}.attention.wv.weight"],
          F"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[F"layers.{layer_i}.attention.wo.weight"],
          F"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w1.weight"],
          F"model.layers.{layer_i}.mlp.down_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w2.weight"],
          F"model.layers.{layer_i}.mlp.up_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w3.weight"],
          F"model.layers.{layer_i}.input_layernorm.weight": loaded[F"layers.{layer_i}.attention_norm.weight"],
          F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[F"layers.{layer_i}.ffn_norm.weight"],
      }
    else:
      # Sharded
      # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
      # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
      # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

      lowercase    :List[str]      	=						{
          F"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
              F"layers.{layer_i}.attention_norm.weight"
          ].clone(),
          F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
              F"layers.{layer_i}.ffn_norm.weight"
          ].clone(),
      }
      lowercase    :str      	=						permute(
          torch.cat(
              [
                  loaded[i][F"layers.{layer_i}.attention.wq.weight"].view(lowerCAmelCase_,			lowerCAmelCase_,			lowerCAmelCase_   )
                  for i in range(lowerCAmelCase_   )
              ],			dim=0,			).reshape(lowerCAmelCase_,			lowerCAmelCase_   )   )
      lowercase    :Tuple      	=						permute(
          torch.cat(
              [
                  loaded[i][F"layers.{layer_i}.attention.wk.weight"].view(
                      lowerCAmelCase_,			lowerCAmelCase_,			lowerCAmelCase_   )
                  for i in range(lowerCAmelCase_   )
              ],			dim=0,			).reshape(lowerCAmelCase_,			lowerCAmelCase_   ),			lowerCAmelCase_,			lowerCAmelCase_,			lowerCAmelCase_,			)
      lowercase    :Dict      	=						torch.cat(
          [
              loaded[i][F"layers.{layer_i}.attention.wv.weight"].view(
                  lowerCAmelCase_,			lowerCAmelCase_,			lowerCAmelCase_   )
              for i in range(lowerCAmelCase_   )
          ],			dim=0,			).reshape(lowerCAmelCase_,			lowerCAmelCase_   )

      lowercase    :Optional[int]      	=						torch.cat(
          [loaded[i][F"layers.{layer_i}.attention.wo.weight"] for i in range(lowerCAmelCase_   )],			dim=1   )
      lowercase    :Dict      	=						torch.cat(
          [loaded[i][F"layers.{layer_i}.feed_forward.w1.weight"] for i in range(lowerCAmelCase_   )],			dim=0   )
      lowercase    :Union[str, Any]      	=						torch.cat(
          [loaded[i][F"layers.{layer_i}.feed_forward.w2.weight"] for i in range(lowerCAmelCase_   )],			dim=1   )
      lowercase    :Dict      	=						torch.cat(
          [loaded[i][F"layers.{layer_i}.feed_forward.w3.weight"] for i in range(lowerCAmelCase_   )],			dim=0   )

    lowercase    :Optional[Any]      	=						inv_freq
    for k, v in state_dict.items():
      lowercase    :List[Any]      	=						filename
      param_count += v.numel()
    torch.save(lowerCAmelCase_,			os.path.join(lowerCAmelCase_,			lowerCAmelCase_   )   )

  lowercase    :Dict      	=						F"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
  if model_size == "7B":
    # Unsharded
    lowercase    :Dict      	=						{
        "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
        "model.norm.weight": loaded["norm.weight"],
        "lm_head.weight": loaded["output.weight"],
    }
  else:
    lowercase    :Any      	=						{
        "model.norm.weight": loaded[0]["norm.weight"],
        "model.embed_tokens.weight": torch.cat(
            [loaded[i]["tok_embeddings.weight"] for i in range(lowerCAmelCase_   )],			dim=1   ),
        "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(lowerCAmelCase_   )],			dim=0   ),
    }

  for k, v in state_dict.items():
    lowercase    :str      	=						filename
    param_count += v.numel()
  torch.save(lowerCAmelCase_,			os.path.join(lowerCAmelCase_,			lowerCAmelCase_   )   )

  # Write configs
  lowercase    :str      	=						{"total_size": param_count * 2}
  write_json(lowerCAmelCase_,			os.path.join(lowerCAmelCase_,			"pytorch_model.bin.index.json"   )   )
  lowercase    :Union[str, Any]      	=						params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
  lowercase    :Any      	=						params["multiple_of"] if "multiple_of" in params else 256
  lowercase    :Optional[int]      	=						LlamaConfig(
      hidden_size=lowerCAmelCase_,			intermediate_size=compute_intermediate_size(lowerCAmelCase_,			lowerCAmelCase_,			lowerCAmelCase_   ),			num_attention_heads=params["n_heads"],			num_hidden_layers=params["n_layers"],			rms_norm_eps=params["norm_eps"],			num_key_value_heads=lowerCAmelCase_,			)
  config.save_pretrained(lowerCAmelCase_   )

  # Make space so we can load the model properly now.
  del state_dict
  del loaded
  gc.collect()

  print("Loading the checkpoint in a Llama model."   )
  lowercase    :List[Any]      	=						LlamaForCausalLM.from_pretrained(lowerCAmelCase_,			torch_dtype=torch.floataa,			low_cpu_mem_usage=lowerCAmelCase_   )
  # Avoid saving this as part of the config.
  del model.config._name_or_path

  print("Saving in the Transformers format."   )
  model.save_pretrained(lowerCAmelCase_,			safe_serialization=lowerCAmelCase_   )
  shutil.rmtree(lowerCAmelCase_   )



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):
  lowercase    :str      	=						LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
  print(F"Saving a {tokenizer_class.__name__} to {tokenizer_path}."   )
  lowercase    :Tuple      	=						tokenizer_class(lowerCAmelCase_   )
  tokenizer.save_pretrained(lowerCAmelCase_   )



def       UpperCAmelCase__  (     ):
  lowercase    :Any      	=						argparse.ArgumentParser()
  parser.add_argument(
      "--input_dir",			help="Location of LLaMA weights, which contains tokenizer.model and model folders",			)
  parser.add_argument(
      "--model_size",			choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"],			)
  parser.add_argument(
      "--output_dir",			help="Location to write HF model and tokenizer",			)
  parser.add_argument("--safe_serialization",			type=lowerCAmelCase_,			help="Whether or not to save using `safetensors`."   )
  lowercase    :Dict      	=						parser.parse_args()
  if args.model_size != "tokenizer_only":
    write_model(
        model_path=args.output_dir,			input_base_path=os.path.join(args.input_dir,			args.model_size   ),			model_size=args.model_size,			safe_serialization=args.safe_serialization,			)
  lowercase    :Tuple      	=						os.path.join(args.input_dir,			"tokenizer.model"   )
  write_tokenizer(args.output_dir,			lowerCAmelCase_   )


if __name__ == "__main__":
     main()









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase	:						Optional[int]			=     logging.get_logger(__name__)

_UpperCAmelCase	:						Union[str, Any]			=     {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_UpperCAmelCase	:						Optional[Any]			=     {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

_UpperCAmelCase	:						Tuple			=     {
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

_UpperCAmelCase	:						Dict			=     {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}




class    __lowerCAmelCase       (							lowerCAmelCase):
       _a 					=						VOCAB_FILES_NAMES
       _a 					=						PRETRAINED_VOCAB_FILES_MAP
       _a 					=						PRETRAINED_INIT_CONFIGURATION
       _a 					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _a 					=						RealmTokenizer






       def __init__(   self:					Tuple       ,					_lowerCAmelCase:					int=None       ,					_lowerCAmelCase:					List[Any]=None       ,					_lowerCAmelCase:					Optional[int]=True       ,					_lowerCAmelCase:					Optional[Any]="[UNK]"       ,					_lowerCAmelCase:					Dict="[SEP]"       ,					_lowerCAmelCase:					List[str]="[PAD]"       ,					_lowerCAmelCase:					Optional[int]="[CLS]"       ,					_lowerCAmelCase:					List[Any]="[MASK]"       ,					_lowerCAmelCase:					Tuple=True       ,					_lowerCAmelCase:					Tuple=None       ,					**_lowerCAmelCase:					Union[str, Any]       ,					):
         super().__init__(
             _lowerCAmelCase       ,					tokenizer_file=_lowerCAmelCase       ,					do_lower_case=_lowerCAmelCase       ,					unk_token=_lowerCAmelCase       ,					sep_token=_lowerCAmelCase       ,					pad_token=_lowerCAmelCase       ,					cls_token=_lowerCAmelCase       ,					mask_token=_lowerCAmelCase       ,					tokenize_chinese_chars=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase       ,					**_lowerCAmelCase       ,					)

         lowercase    :List[str]      	=						json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
         if (
             normalizer_state.get("lowercase"       ,					_lowerCAmelCase				) != do_lower_case
             or normalizer_state.get("strip_accents"       ,					_lowerCAmelCase				) != strip_accents
             or normalizer_state.get("handle_chinese_chars"       ,					_lowerCAmelCase				) != tokenize_chinese_chars
         ):
           lowercase    :List[str]      	=						getattr(_lowerCAmelCase       ,					normalizer_state.pop("type"				)				)
           lowercase    :List[str]      	=						do_lower_case
           lowercase    :Any      	=						strip_accents
           lowercase    :Tuple      	=						tokenize_chinese_chars
           lowercase    :Any      	=						normalizer_class(**_lowerCAmelCase				)

         lowercase    :Any      	=						do_lower_case






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Optional[Any]       ,					**_lowerCAmelCase:					Optional[int]				):
         lowercase    :List[str]      	=						PaddingStrategy.MAX_LENGTH

         lowercase    :Optional[Any]      	=						text
         lowercase    :str      	=						kwargs.pop("text_pair"       ,					_lowerCAmelCase				)
         lowercase    :Any      	=						kwargs.pop("return_tensors"       ,					_lowerCAmelCase				)

         lowercase    :List[str]      	=						{
             "input_ids": [],
             "attention_mask": [],
             "token_type_ids": [],
         }

         for idx, candidate_text in enumerate(_lowerCAmelCase				):
           if batch_text_pair is not None:
             lowercase    :Any      	=						batch_text_pair[idx]
           else:
             lowercase    :Optional[Any]      	=						None

           lowercase    :Optional[Any]      	=						super().__call__(_lowerCAmelCase       ,					_lowerCAmelCase       ,					return_tensors=_lowerCAmelCase       ,					**_lowerCAmelCase				)

           lowercase    :Tuple      	=						encoded_candidates.get("input_ids"				)
           lowercase    :Optional[Any]      	=						encoded_candidates.get("attention_mask"				)
           lowercase    :int      	=						encoded_candidates.get("token_type_ids"				)

           if encoded_input_ids is not None:
             output_data["input_ids"].append(_lowerCAmelCase				)
           if encoded_attention_mask is not None:
             output_data["attention_mask"].append(_lowerCAmelCase				)
           if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(_lowerCAmelCase				)

         lowercase    :str      	=						{key: item for key, item in output_data.items() if len(_lowerCAmelCase				) != 0}

         return BatchEncoding(_lowerCAmelCase       ,					tensor_type=_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Any       ,					_lowerCAmelCase:					Tuple=None				):
         lowercase    :Dict      	=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

         return output






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple       ,					_lowerCAmelCase:					List[int]       ,					_lowerCAmelCase:					Optional[List[int]] = None				):
         lowercase    :List[str]      	=						[self.sep_token_id]
         lowercase    :List[Any]      	=						[self.cls_token_id]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]       ,					_lowerCAmelCase:					str       ,					_lowerCAmelCase:					Optional[str] = None				):
         lowercase    :Tuple      	=						self._tokenizer.model.save(_lowerCAmelCase       ,					name=_lowerCAmelCase				)
         return tuple(_lowerCAmelCase				)



