Update configuration_llava.py

configuration_llava.py

@@ -1,18 +1,107 @@
-# coding=utf-8
+rom transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+from transformers import SiglipVisionConfig
 
-from transformers.configuration_utils import PretrainedConfig
-from open_clip import get_model_config
-from configuration_phi import PhiConfig
+
+logger = logging.get_logger(__name__)
+
+
+class PhiConfig(PretrainedConfig):
+    model_type = "phi"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=51200,
+        hidden_size=2048,
+        intermediate_size=8192,
+        num_hidden_layers=24,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        resid_pdrop=0.0,
+        embd_pdrop=0.0,
+        attention_dropout=0.0,
+        hidden_act="gelu_new",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        layer_norm_eps=1e-5,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        partial_rotary_factor=0.5,
+        qk_layernorm=False,
+        bos_token_id=1,
+        eos_token_id=2,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attention_dropout = attention_dropout
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.partial_rotary_factor = partial_rotary_factor
+        self.qk_layernorm = qk_layernorm
+        self._rope_scaling_validation()
+
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def _rope_scaling_validation(self):
+        """
+        Validate the `rope_scaling` configuration.
+        """
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
+            raise ValueError(
+                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
+                f"got {self.rope_scaling}"
+            )
+        rope_scaling_type = self.rope_scaling.get("type", None)
+        rope_scaling_factor = self.rope_scaling.get("factor", None)
+        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
+            raise ValueError(
+                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
+            )
+        if (
+            rope_scaling_factor is None
+            or not isinstance(rope_scaling_factor, float)
+            or rope_scaling_factor <= 1.0
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}"
+            )
 
 
 class LlavaConfig(PretrainedConfig):
-    model_type = "llava"
+    model_type = "HelpingAI-V"
     is_composition = False
 
     def __init__(
         self,
         text_config=None,
-        vision_tower_name="ViT-SO400M-14-SigLIP-384",
+        vision_config=None,
         ignore_index=-100,
         image_token_index=50297,
         projector_hidden_act="gelu",
@@ -26,16 +115,17 @@ class LlavaConfig(PretrainedConfig):
         self.projector_tokens_num = projector_tokens_num
         self.vocab_size = vocab_size
 
-        self.vision_tower_name = vision_tower_name
-        vision_config = get_model_config(vision_tower_name)
-        self.vision_embed_dim = vision_config["embed_dim"]
-
-        self.vocab_size = self.vocab_size
-
         self.text_config = text_config
         if isinstance(self.text_config, dict):
-            text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "llama"
+            text_config["model_type"] = (
+                text_config["model_type"] if "model_type" in text_config else "phi"
+            )
             self.text_config = PhiConfig(**text_config)
             self.vocab_size = self.text_config.vocab_size
 
-        super().__init__(**kwargs)
+        self.vision_config = vision_config
+        if isinstance(self.vision_config, dict):
+            self.vision_config = SiglipVisionConfig(**vision_config)
+            self.vision_embed_dim = self.vision_config.hidden_size
+
+        super().__init__(**kwargs)