Update modeling_phi.py

modeling_phi.py

@@ -296,8 +296,8 @@ class MoE(nn.Module):
         config: PretrainedConfig,
     ):
         super().__init__()
-        self.mlp = nn.ModuleList([MLP(config) for i in range(config.num_experts)])
-        self.gate = nn.Linear(config.n_embd, config.num_experts, bias=False)
+        self.mlp = nn.ModuleList([MLP(config) for i in range(config.num_local_experts)])
+        self.gate = nn.Linear(config.n_embd, config.num_local_experts, bias=False)
         self.num_experts_per_tok = config.num_experts_per_tok
 
     def forward(self, x):