--- library_name: transformers pipeline_tag: text-generation inference: true widget: - text: Hello! example_title: Hello world group: Python base_model: - microsoft/Phi-4-mini-flash-reasoning --- This tiny model is for debugging. It is randomly initialized with the config adapted from [microsoft/Phi-4-mini-flash-reasoning](https://huggingface.co/microsoft/Phi-4-mini-flash-reasoning). ### Example usage: ```python import torch from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline torch.random.manual_seed(0) model_id = "yujiepan/phi-4-flash-tiny-random" model = AutoModelForCausalLM.from_pretrained( model_id, device_map="cuda", torch_dtype=torch.bfloat16, trust_remote_code=True, ) tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) messages = [{ "role": "user", "content": "How to solve 3*x^2+4*x+5=1?" }] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, return_dict=True, return_tensors="pt", ) outputs = model.generate( **inputs.to(model.device), max_new_tokens=600, temperature=0.6, top_p=0.95, do_sample=True, ) outputs = tokenizer.batch_decode(outputs[:, inputs["input_ids"].shape[-1]:]) print(outputs[0]) ``` ### Codes to create this repo: ```python import json from pathlib import Path import accelerate import torch from huggingface_hub import file_exists, hf_hub_download from transformers import ( AutoConfig, AutoModelForCausalLM, AutoProcessor, GenerationConfig, set_seed, ) source_model_id = "microsoft/Phi-4-mini-flash-reasoning" save_folder = "/tmp/yujiepan/phi-4-flash-tiny-random" processor = AutoProcessor.from_pretrained(source_model_id, trust_remote_code=True) processor.save_pretrained(save_folder) with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f: config_json = json.load(f) for key in ['AutoConfig', 'AutoModelForCausalLM']: config_json['auto_map'][key] = f'{source_model_id}--' + config_json['auto_map'][key] automap = config_json['auto_map'] config_json['hidden_size'] = 64 config_json['intermediate_size'] = 64 config_json['num_attention_heads'] = 2 config_json['num_hidden_layers'] = 4 config_json['num_key_value_heads'] = 2 config_json['tie_word_embeddings'] = True config_json['sliding_window'] = 512 config_json['use_cache'] = True config_json['mb_per_layer'] = 2 # first layer is mamba with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f: json.dump(config_json, f, indent=2) config = AutoConfig.from_pretrained( save_folder, trust_remote_code=True, ) print(config) torch.set_default_dtype(torch.bfloat16) model = AutoModelForCausalLM.from_config(config, trust_remote_code=True) torch.set_default_dtype(torch.float32) if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'): model.generation_config = GenerationConfig.from_pretrained( source_model_id, trust_remote_code=True, ) set_seed(42) model = model.cpu() # cpu is more stable for random initialization across machines with torch.no_grad(): for name, p in sorted(model.named_parameters()): torch.nn.init.normal_(p, 0, 0.2) print(name, p.shape) model.save_pretrained(save_folder) print(model) with open(f"{save_folder}/config.json", "r", encoding='utf-8') as f: config_json = json.load(f) config_json['auto_map'] = automap config_json['sliding_window'] = 512 # a bugfix for '<' not supported between instances of 'int' and 'list' with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f: json.dump(config_json, f, indent=2) for python_file in Path(save_folder).glob('*.py'): if python_file.name.startswith('modeling_') or python_file.name.startswith('configuration_'): python_file.unlink() ``` ### Printing the model: ```text Phi4FlashForCausalLM( (model): Phi4FlashModel( (embed_tokens): Embedding(200064, 64, padding_idx=199999) (embed_dropout): Dropout(p=0.0, inplace=False) (layers): ModuleList( (0): SambaYDecoderLayer( (mlp): SambaYMLP( (fc1): Linear(in_features=64, out_features=128, bias=False) (fc2): Linear(in_features=64, out_features=64, bias=False) (activation_fn): SiLU() ) (input_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) (attn): Phi3Mamba( (in_proj): Linear(in_features=64, out_features=256, bias=False) (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128) (act): SiLU() (x_proj): Linear(in_features=128, out_features=36, bias=False) (dt_proj): Linear(in_features=4, out_features=128, bias=True) (out_proj): Linear(in_features=128, out_features=64, bias=False) ) (resid_attn_dropout): Dropout(p=0.0, inplace=False) (resid_mlp_dropout): Dropout(p=0.0, inplace=False) (post_attention_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) ) (1): SambaYDecoderLayer( (mlp): SambaYMLP( (fc1): Linear(in_features=64, out_features=128, bias=False) (fc2): Linear(in_features=64, out_features=64, bias=False) (activation_fn): SiLU() ) (input_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) (attn): SambaYFlashAttention2( (out_proj): Linear(in_features=64, out_features=64, bias=True) (Wqkv): Linear(in_features=64, out_features=192, bias=True) (inner_cross_attn): FlashDiffCustomAttention( (subln): SambaYRMSNorm() ) ) (resid_attn_dropout): Dropout(p=0.0, inplace=False) (resid_mlp_dropout): Dropout(p=0.0, inplace=False) (post_attention_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) ) (2): SambaYDecoderLayer( (mlp): SambaYMLP( (fc1): Linear(in_features=64, out_features=128, bias=False) (fc2): Linear(in_features=64, out_features=64, bias=False) (activation_fn): SiLU() ) (input_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) (attn): Phi3Mamba( (in_proj): Linear(in_features=64, out_features=256, bias=False) (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128) (act): SiLU() (x_proj): Linear(in_features=128, out_features=36, bias=False) (dt_proj): Linear(in_features=4, out_features=128, bias=True) (out_proj): Linear(in_features=128, out_features=64, bias=False) ) (resid_attn_dropout): Dropout(p=0.0, inplace=False) (resid_mlp_dropout): Dropout(p=0.0, inplace=False) (post_attention_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) ) (3): SambaYDecoderLayer( (mlp): SambaYMLP( (fc1): Linear(in_features=64, out_features=128, bias=False) (fc2): Linear(in_features=64, out_features=64, bias=False) (activation_fn): SiLU() ) (input_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) (attn): SambaYFlashAttention2( (out_proj): Linear(in_features=64, out_features=64, bias=True) (Wqkv): Linear(in_features=64, out_features=192, bias=True) (inner_cross_attn): FlashDiffCustomAttention( (subln): SambaYRMSNorm() ) ) (resid_attn_dropout): Dropout(p=0.0, inplace=False) (resid_mlp_dropout): Dropout(p=0.0, inplace=False) (post_attention_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) ) ) (final_layernorm): LayerNorm((64,), eps=1e-05, elementwise_affine=True) ) (lm_head): Linear(in_features=64, out_features=200064, bias=False) ) ```