





from typing import Optional, Union

import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models.modeling_utils import ModelMixin





class      lowercase__    (							__A     ,					__A  ):







  @register_to_config
  def __init__(	self    ,						_lowercase = 768    ,						):
       super().__init__()

       lowerCAmelCase_					:       Optional[int]    =		nn.Parameter(torch.zeros(1    ,						_lowercase						)						)
       lowerCAmelCase_					:       Dict    =		nn.Parameter(torch.ones(1    ,						_lowercase						)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase = None    ,						_lowercase = None    ,						):
       lowerCAmelCase_					:       Any    =		nn.Parameter(self.mean.to(_lowercase						).to(_lowercase						)						)
       lowerCAmelCase_					:       str    =		nn.Parameter(self.std.to(_lowercase						).to(_lowercase						)						)
       return self







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[Any]    =		(embeds - self.mean) * 1.0 / self.std
       return embeds







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       Tuple    =		(embeds * self.std) + self.mean
       return embeds







import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel





class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  """M-CLIP"""







  def __init__(	self    ,						_lowercase=1_024    ,						_lowercase=768    ,						**_lowercase						):
       lowerCAmelCase_					:       Tuple    =		transformerDimSize
       lowerCAmelCase_					:       Optional[Any]    =		imageDimSize
       super().__init__(**_lowercase						)





class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  MCLIPConfig







  def __init__(	self    ,						_lowercase    ,						*_lowercase    ,						**_lowercase						):
       super().__init__(_lowercase    ,						*_lowercase    ,						**_lowercase						)
       lowerCAmelCase_					:       str    =		XLMRobertaModel(_lowercase						)
       lowerCAmelCase_					:       str    =		torch.nn.Linear(
           in_features=config.transformerDimensions    ,						out_features=config.numDims						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[int]    =		self.transformer(input_ids=_lowercase    ,						attention_mask=_lowercase						)[0]
       lowerCAmelCase_					:       Any    =		(embs * attention_mask.unsqueeze(2						)).sum(dim=1						) / attention_mask.sum(dim=1						)[:, None]
       return self.LinearTransformation(_lowercase						), embs
