'''simple docstring'''

import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
           DistilBertForMaskedLM,
           DistilBertForMultipleChoice,
           DistilBertForQuestionAnswering,
           DistilBertForSequenceClassification,
           DistilBertForTokenClassification,
           DistilBertModel,
       )





class 							lowercase						(						__lowerCamelCase ):


   def __init__(   self				:	Optional[Any]   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	Dict=13   ,		__lowerCAmelCase				:	Optional[int]=7   ,		__lowerCAmelCase				:	Union[str, Any]=True   ,		__lowerCAmelCase				:	List[str]=True   ,		__lowerCAmelCase				:	List[str]=False   ,		__lowerCAmelCase				:	Dict=True   ,		__lowerCAmelCase				:	Dict=99   ,		__lowerCAmelCase				:	Optional[Any]=32   ,		__lowerCAmelCase				:	int=5   ,		__lowerCAmelCase				:	int=4   ,		__lowerCAmelCase				:	Dict=37   ,		__lowerCAmelCase				:	int="gelu"   ,		__lowerCAmelCase				:	Any=0.1   ,		__lowerCAmelCase				:	List[Any]=0.1   ,		__lowerCAmelCase				:	Any=512   ,		__lowerCAmelCase				:	str=16   ,		__lowerCAmelCase				:	str=2   ,		__lowerCAmelCase				:	Union[str, Any]=0.02   ,		__lowerCAmelCase				:	Tuple=3   ,		__lowerCAmelCase				:	List[Any]=4   ,		__lowerCAmelCase				:	Union[str, Any]=None   ,		)      ->					Any:
         lowercase_     =      parent
         lowercase_     =      batch_size
         lowercase_     =      seq_length
         lowercase_     =      is_training
         lowercase_     =      use_input_mask
         lowercase_     =      use_token_type_ids
         lowercase_     =      use_labels
         lowercase_     =      vocab_size
         lowercase_     =      hidden_size
         lowercase_     =      num_hidden_layers
         lowercase_     =      num_attention_heads
         lowercase_     =      intermediate_size
         lowercase_     =      hidden_act
         lowercase_     =      hidden_dropout_prob
         lowercase_     =      attention_probs_dropout_prob
         lowercase_     =      max_position_embeddings
         lowercase_     =      type_vocab_size
         lowercase_     =      type_sequence_label_size
         lowercase_     =      initializer_range
         lowercase_     =      num_labels
         lowercase_     =      num_choices
         lowercase_     =      scope


   def __UpperCAmelCase       (   self				:	Union[str, Any])      ->					str:
         lowercase_     =      ids_tensor([self.batch_size, self.seq_length]   ,		self.vocab_size)

         lowercase_     =      None
         if self.use_input_mask:
               lowercase_     =      random_attention_mask([self.batch_size, self.seq_length])

         lowercase_     =      None
         lowercase_     =      None
         lowercase_     =      None
         if self.use_labels:
               lowercase_     =      ids_tensor([self.batch_size]   ,		self.type_sequence_label_size)
               lowercase_     =      ids_tensor([self.batch_size, self.seq_length]   ,		self.num_labels)
               lowercase_     =      ids_tensor([self.batch_size]   ,		self.num_choices)

         lowercase_     =      self.get_config()

         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


   def __UpperCAmelCase       (   self				:	str)      ->					Union[str, Any]:
         return DistilBertConfig(
             vocab_size=self.vocab_size   ,		dim=self.hidden_size   ,		n_layers=self.num_hidden_layers   ,		n_heads=self.num_attention_heads   ,		hidden_dim=self.intermediate_size   ,		hidden_act=self.hidden_act   ,		dropout=self.hidden_dropout_prob   ,		attention_dropout=self.attention_probs_dropout_prob   ,		max_position_embeddings=self.max_position_embeddings   ,		initializer_range=self.initializer_range   ,		)


   def __UpperCAmelCase       (   self				:	Optional[Any]   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	Optional[int]   ,		__lowerCAmelCase				:	Any   ,		__lowerCAmelCase				:	str   ,		__lowerCAmelCase				:	Optional[Any]   ,		__lowerCAmelCase				:	Dict)      ->					Dict:
         lowercase_     =      DistilBertModel(config=__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()
         lowercase_     =      model(__lowerCAmelCase   ,		__lowerCAmelCase)
         lowercase_     =      model(__lowerCAmelCase)
         self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.hidden_size))


   def __UpperCAmelCase       (   self				:	str   ,		__lowerCAmelCase				:	List[str]   ,		__lowerCAmelCase				:	Optional[int]   ,		__lowerCAmelCase				:	Any   ,		__lowerCAmelCase				:	Optional[int]   ,		__lowerCAmelCase				:	Optional[int]   ,		__lowerCAmelCase				:	Any)      ->					List[Any]:
         lowercase_     =      DistilBertForMaskedLM(config=__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()
         lowercase_     =      model(__lowerCAmelCase   ,		attention_mask=__lowerCAmelCase   ,		labels=__lowerCAmelCase)
         self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size))


   def __UpperCAmelCase       (   self				:	List[str]   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	List[str]   ,		__lowerCAmelCase				:	int   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	str   ,		__lowerCAmelCase				:	List[str])      ->					Any:
         lowercase_     =      DistilBertForQuestionAnswering(config=__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()
         lowercase_     =      model(
             __lowerCAmelCase   ,		attention_mask=__lowerCAmelCase   ,		start_positions=__lowerCAmelCase   ,		end_positions=__lowerCAmelCase)
         self.parent.assertEqual(result.start_logits.shape   ,		(self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape   ,		(self.batch_size, self.seq_length))


   def __UpperCAmelCase       (   self				:	Any   ,		__lowerCAmelCase				:	List[Any]   ,		__lowerCAmelCase				:	Optional[Any]   ,		__lowerCAmelCase				:	Tuple   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	Union[str, Any]   ,		__lowerCAmelCase				:	Tuple)      ->					List[Any]:
         lowercase_     =      self.num_labels
         lowercase_     =      DistilBertForSequenceClassification(__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()
         lowercase_     =      model(__lowerCAmelCase   ,		attention_mask=__lowerCAmelCase   ,		labels=__lowerCAmelCase)
         self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_labels))


   def __UpperCAmelCase       (   self				:	Optional[Any]   ,		__lowerCAmelCase				:	Any   ,		__lowerCAmelCase				:	Optional[Any]   ,		__lowerCAmelCase				:	str   ,		__lowerCAmelCase				:	Tuple   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	int)      ->					Union[str, Any]:
         lowercase_     =      self.num_labels
         lowercase_     =      DistilBertForTokenClassification(config=__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()

         lowercase_     =      model(__lowerCAmelCase   ,		attention_mask=__lowerCAmelCase   ,		labels=__lowerCAmelCase)
         self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.num_labels))


   def __UpperCAmelCase       (   self				:	Optional[Any]   ,		__lowerCAmelCase				:	Union[str, Any]   ,		__lowerCAmelCase				:	Any   ,		__lowerCAmelCase				:	Optional[int]   ,		__lowerCAmelCase				:	Tuple   ,		__lowerCAmelCase				:	int   ,		__lowerCAmelCase				:	str)      ->					Any:
         lowercase_     =      self.num_choices
         lowercase_     =      DistilBertForMultipleChoice(config=__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()
         lowercase_     =      input_ids.unsqueeze(1).expand(-1   ,		self.num_choices   ,		-1).contiguous()
         lowercase_     =      input_mask.unsqueeze(1).expand(-1   ,		self.num_choices   ,		-1).contiguous()
         lowercase_     =      model(
             __lowerCAmelCase   ,		attention_mask=__lowerCAmelCase   ,		labels=__lowerCAmelCase   ,		)
         self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_choices))


   def __UpperCAmelCase       (   self				:	int)      ->					int:
         lowercase_     =      self.prepare_config_and_inputs()
         ((lowercase_) ,							(lowercase_) ,							(lowercase_) ,							(lowercase_) ,							(lowercase_) ,							(lowercase_))     =      config_and_inputs
         lowercase_     =      {"input_ids": input_ids, "attention_mask": input_mask}
         return config, inputs_dict





@require_torch
class 							lowercase						(						__lowerCamelCase ,				__lowerCamelCase ,				unittest.TestCase ):
   lowerCamelCase_    					=(
       (
           DistilBertModel,
           DistilBertForMaskedLM,
           DistilBertForMultipleChoice,
           DistilBertForQuestionAnswering,
           DistilBertForSequenceClassification,
           DistilBertForTokenClassification,
       )
       if is_torch_available()
       else None
   )
   lowerCamelCase_    					=(
       {
           'feature-extraction': DistilBertModel,
           'fill-mask': DistilBertForMaskedLM,
           'question-answering': DistilBertForQuestionAnswering,
           'text-classification': DistilBertForSequenceClassification,
           'token-classification': DistilBertForTokenClassification,
           'zero-shot': DistilBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   lowerCamelCase_    					=True
   lowerCamelCase_    					=True
   lowerCamelCase_    					=True
   lowerCamelCase_    					=True


   def __UpperCAmelCase       (   self				:	Any)      ->					Optional[Any]:
         lowercase_     =      DistilBertModelTester(self)
         lowercase_     =      ConfigTester(self   ,		config_class=__lowerCAmelCase   ,		dim=37)


   def __UpperCAmelCase       (   self				:	Any)      ->					List[Any]:
         self.config_tester.run_common_tests()


   def __UpperCAmelCase       (   self				:	Dict)      ->					Optional[Any]:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_distilbert_model(*__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	Optional[int])      ->					Optional[Any]:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_distilbert_for_masked_lm(*__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	Any)      ->					Optional[int]:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_distilbert_for_question_answering(*__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	Any)      ->					List[str]:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_distilbert_for_sequence_classification(*__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	List[Any])      ->					str:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_distilbert_for_token_classification(*__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	Any)      ->					List[Any]:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_distilbert_for_multiple_choice(*__lowerCAmelCase)


   @slow
   def __UpperCAmelCase       (   self				:	Tuple)      ->					Optional[int]:
         for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               lowercase_     =      DistilBertModel.from_pretrained(__lowerCAmelCase)
               self.assertIsNotNone(__lowerCAmelCase)


   @slow
   @require_torch_gpu
   def __UpperCAmelCase       (   self				:	Tuple)      ->					int:
         lowercase_ ,							lowercase_     =      self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
               # BertForMultipleChoice behaves incorrectly in JIT environments.
               if model_class == DistilBertForMultipleChoice:
                     return

               lowercase_     =      True
               lowercase_     =      model_class(config=__lowerCAmelCase)

               lowercase_     =      self._prepare_for_class(__lowerCAmelCase   ,		__lowerCAmelCase)
               lowercase_     =      torch.jit.trace(
                   __lowerCAmelCase   ,		(inputs_dict["input_ids"].to("cpu"), inputs_dict["attention_mask"].to("cpu")))

               with tempfile.TemporaryDirectory() as tmp:
                     torch.jit.save(__lowerCAmelCase   ,		os.path.join(__lowerCAmelCase   ,		"traced_model.pt"))
                     lowercase_     =      torch.jit.load(os.path.join(__lowerCAmelCase   ,		"traced_model.pt")   ,		map_location=__lowerCAmelCase)
                     loaded(inputs_dict["input_ids"].to(__lowerCAmelCase)   ,		inputs_dict["attention_mask"].to(__lowerCAmelCase))





@require_torch
class 							lowercase						(						unittest.TestCase ):


   @slow
   def __UpperCAmelCase       (   self				:	str)      ->					Dict:
         lowercase_     =      DistilBertModel.from_pretrained("distilbert-base-uncased")
         lowercase_     =      torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
         lowercase_     =      torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
         with torch.no_grad():
               lowercase_     =      model(__lowerCAmelCase   ,		attention_mask=__lowerCAmelCase)[0]
         lowercase_     =      torch.Size((1, 11, 768))
         self.assertEqual(output.shape   ,		__lowerCAmelCase)
         lowercase_     =      torch.tensor(
             [[[-0.1639, 0.3299, 0.1648], [-0.1746, 0.3289, 0.1710], [-0.1884, 0.3357, 0.1810]]])

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4]   ,		__lowerCAmelCase   ,		atol=1e-4))







'''simple docstring'''

import argparse
import os

import torch

from transformers import FlavaConfig, FlavaForPreTraining
from transformers.models.flava.convert_dalle_to_flava_codebook import convert_dalle_checkpoint


def __a (	__lowerCamelCase			:						List[Any]				)	->							Tuple:
      '''simple docstring'''


      return sum(param.float().sum() if "encoder.embeddings" not in key else 0 for key, param in state_dict.items()				)


def __a (	__lowerCamelCase			:						str ,		__lowerCamelCase			:						str				)	->							List[str]:
      '''simple docstring'''


      lowercase_     =      {}

      for key, value in state_dict.items():
            if "text_encoder.embeddings" in key or "image_encoder.embeddings" in key:
                  continue

            lowercase_     =      key.replace("heads.cmd.mim_head.cls.predictions" ,		"mmm_image_head"				)
            lowercase_     =      key.replace("heads.cmd.mlm_head.cls.predictions" ,		"mmm_text_head"				)
            lowercase_     =      key.replace("heads.cmd.itm_head.cls" ,		"itm_head"				)
            lowercase_     =      key.replace("heads.cmd.itm_head.pooler" ,		"itm_head.pooler"				)
            lowercase_     =      key.replace("heads.cmd.clip_head.logit_scale" ,		"flava.logit_scale"				)
            lowercase_     =      key.replace("heads.fairseq_mlm.cls.predictions" ,		"mlm_head"				)
            lowercase_     =      key.replace("heads.imagenet.mim_head.cls.predictions" ,		"mim_head"				)
            lowercase_     =      key.replace("mm_text_projection" ,		"flava.text_to_mm_projection"				)
            lowercase_     =      key.replace("mm_image_projection" ,		"flava.image_to_mm_projection"				)
            lowercase_     =      key.replace("image_encoder.module" ,		"flava.image_model"				)
            lowercase_     =      key.replace("text_encoder.module" ,		"flava.text_model"				)
            lowercase_     =      key.replace("mm_encoder.module.encoder.cls_token" ,		"flava.multimodal_model.cls_token"				)
            lowercase_     =      key.replace("mm_encoder.module" ,		"flava.multimodal_model"				)
            lowercase_     =      key.replace("text_projection" ,		"flava.text_projection"				)
            lowercase_     =      key.replace("image_projection" ,		"flava.image_projection"				)

            lowercase_     =      value.float()

      for key, value in codebook_state_dict.items():
            lowercase_     =      value

      return upgrade


@torch.no_grad()
def __a (	__lowerCamelCase			:						Dict ,		__lowerCamelCase			:						List[Any] ,		__lowerCamelCase			:						List[str] ,		__lowerCamelCase			:						Tuple=None				)	->							Optional[int]:
      '''simple docstring'''


      if config_path is not None:
            lowercase_     =      FlavaConfig.from_pretrained(__lowerCamelCase				)
      else:
            lowercase_     =      FlavaConfig()

      lowercase_     =      FlavaForPreTraining(__lowerCamelCase				).eval()

      lowercase_     =      convert_dalle_checkpoint(__lowerCamelCase ,		__lowerCamelCase ,		save_checkpoint=__lowerCamelCase				)

      if os.path.exists(__lowerCamelCase				):
            lowercase_     =      torch.load(__lowerCamelCase ,		map_location="cpu"				)
      else:
            lowercase_     =      torch.hub.load_state_dict_from_url(__lowerCamelCase ,		map_location="cpu"				)

      lowercase_     =      upgrade_state_dict(__lowerCamelCase ,		__lowerCamelCase				)
      hf_model.load_state_dict(__lowerCamelCase				)
      lowercase_     =      hf_model.state_dict()
      lowercase_     =      count_parameters(__lowerCamelCase				)
      lowercase_     =      count_parameters(__lowerCamelCase				) + count_parameters(__lowerCamelCase				)

      assert torch.allclose(__lowerCamelCase ,		__lowerCamelCase ,		atol=1e-3				)

      hf_model.save_pretrained(__lowerCamelCase				)


if __name__ == "__main__":
       lowerCAmelCase_ : List[str]						  =      argparse.ArgumentParser()
       parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
       parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to flava checkpoint")
       parser.add_argument("--codebook_path", default=None, type=str, help="Path to flava codebook checkpoint")
       parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert")
       lowerCAmelCase_ : List[Any]						  =      parser.parse_args()

       convert_flava_checkpoint(args.checkpoint_path, args.codebook_path, args.pytorch_dump_folder_path, args.config_path)






