

'''simple docstring'''





import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
__UpperCAmelCase    				=     logging.getLogger()

__UpperCAmelCase    				=     logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)





class A__    ( UpperCAmelCase_ ):



 """simple docstring"""







 def    __magic_name__       (		self :	Optional[int]     ,						A_ :	int       ):






      '''simple docstring'''




      os.makedirs(_snake_case     ,						exist_ok=_snake_case       )
      _lowerCAmelCase   :					Optional[Any]        =    {"source": "What is love ?", "target": "life"}
      _lowerCAmelCase   :					Optional[Any]        =    {"train": 1_2, "val": 2, "test": 2}
      for split in ["train", "test", "val"]:
           for field in ["source", "target"]:
                _lowerCAmelCase   :					Optional[Any]        =    "\n".join([contents[field]] * n_lines[split]       )
                with open(os.path.join(_snake_case     ,						F'''{split}.{field}'''       )     ,						"w"       ) as f:
                     f.write(_snake_case       )

 def    __magic_name__       (		self :	Union[str, Any]     ,						A_ :	int     ,						A_ :	str = "pytorch"       ):






      '''simple docstring'''




      _lowerCAmelCase   :					List[Any]        =    self.get_auto_remove_tmp_dir()
      _lowerCAmelCase   :					Union[str, Any]        =    os.path.join(_snake_case     ,						"output"       )
      _lowerCAmelCase   :					Any        =    os.path.join(_snake_case     ,						"data"       )
      self._create_dummy_data(data_dir=_snake_case       )

      _lowerCAmelCase   :					List[Any]        =    F'''
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            '''.split()

      if gpus > 0:
           testargs.append(F'''--gpus={gpus}'''       )
           if is_apex_available():
                testargs.append("--fp16"       )
      else:
           testargs.append("--gpus=0"       )
           testargs.append("--distributed_backend=ddp_cpu"       )
           testargs.append("--num_processes=2"       )

      _lowerCAmelCase   :					int        =    [sys.executable, str(Path(finetune_rag.__file__       ).resolve()       )] + testargs
      execute_subprocess_async(_snake_case     ,						env=self.get_env()       )

      _lowerCAmelCase   :					Union[str, Any]        =    os.path.join(_snake_case     ,						"metrics.json"       )
      with open(_snake_case       ) as f:
           _lowerCAmelCase   :					Optional[int]        =    json.load(_snake_case       )
      return result

 @require_torch_gpu
 def    __magic_name__       (		self :	Dict       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[int]        =    self._run_finetune(gpus=1       )
      self.assertGreaterEqual(result["test"][0]["test_avg_em"]     ,						0.2       )

 @require_torch_multi_gpu
 def    __magic_name__       (		self :	Tuple       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Dict        =    self._run_finetune(gpus=2       )
      self.assertGreaterEqual(result["test"][0]["test_avg_em"]     ,						0.2       )

 @require_torch_gpu
 @require_ray
 def    __magic_name__       (		self :	List[Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					List[Any]        =    self._run_finetune(gpus=1     ,						distributed_retriever="ray"       )
      self.assertGreaterEqual(result["test"][0]["test_avg_em"]     ,						0.2       )


 @require_torch_multi_gpu
 @require_ray
 def    __magic_name__       (		self :	Optional[int]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[Any]        =    self._run_finetune(gpus=1     ,						distributed_retriever="ray"       )
      self.assertGreaterEqual(result["test"][0]["test_avg_em"]     ,						0.2       )









import argparse
import os

import torch

from transformers import FlavaConfig, FlavaForPreTraining
from transformers.models.flava.convert_dalle_to_flava_codebook import convert_dalle_checkpoint







def 	_snake_case		(    SCREAMING_SNAKE_CASE				)       ->      Optional[int]:

     """simple docstring"""







     return sum(param.float().sum() if "encoder.embeddings" not in key else 0 for key, param in state_dict.items()				)







def 	_snake_case		(    SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE				)       ->      Tuple:

     """simple docstring"""







     _lowerCAmelCase   :					str        =    {}

     for key, value in state_dict.items():
          if "text_encoder.embeddings" in key or "image_encoder.embeddings" in key:
               continue

          _lowerCAmelCase   :					Union[str, Any]        =    key.replace("heads.cmd.mim_head.cls.predictions"		,							"mmm_image_head"				)
          _lowerCAmelCase   :					int        =    key.replace("heads.cmd.mlm_head.cls.predictions"		,							"mmm_text_head"				)
          _lowerCAmelCase   :					Any        =    key.replace("heads.cmd.itm_head.cls"		,							"itm_head"				)
          _lowerCAmelCase   :					Optional[Any]        =    key.replace("heads.cmd.itm_head.pooler"		,							"itm_head.pooler"				)
          _lowerCAmelCase   :					Any        =    key.replace("heads.cmd.clip_head.logit_scale"		,							"flava.logit_scale"				)
          _lowerCAmelCase   :					List[Any]        =    key.replace("heads.fairseq_mlm.cls.predictions"		,							"mlm_head"				)
          _lowerCAmelCase   :					Union[str, Any]        =    key.replace("heads.imagenet.mim_head.cls.predictions"		,							"mim_head"				)
          _lowerCAmelCase   :					Optional[int]        =    key.replace("mm_text_projection"		,							"flava.text_to_mm_projection"				)
          _lowerCAmelCase   :					int        =    key.replace("mm_image_projection"		,							"flava.image_to_mm_projection"				)
          _lowerCAmelCase   :					List[str]        =    key.replace("image_encoder.module"		,							"flava.image_model"				)
          _lowerCAmelCase   :					Optional[int]        =    key.replace("text_encoder.module"		,							"flava.text_model"				)
          _lowerCAmelCase   :					List[str]        =    key.replace("mm_encoder.module.encoder.cls_token"		,							"flava.multimodal_model.cls_token"				)
          _lowerCAmelCase   :					Dict        =    key.replace("mm_encoder.module"		,							"flava.multimodal_model"				)
          _lowerCAmelCase   :					Dict        =    key.replace("text_projection"		,							"flava.text_projection"				)
          _lowerCAmelCase   :					Optional[Any]        =    key.replace("image_projection"		,							"flava.image_projection"				)

          _lowerCAmelCase   :					List[Any]        =    value.float()

     for key, value in codebook_state_dict.items():
          _lowerCAmelCase   :					int        =    value

     return upgrade







@torch.no_grad()
def 	_snake_case		(    SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE=None				)       ->      Tuple:

     """simple docstring"""







     if config_path is not None:
          _lowerCAmelCase   :					Dict        =    FlavaConfig.from_pretrained(SCREAMING_SNAKE_CASE				)
     else:
          _lowerCAmelCase   :					Dict        =    FlavaConfig()

     _lowerCAmelCase   :					List[Any]        =    FlavaForPreTraining(SCREAMING_SNAKE_CASE				).eval()

     _lowerCAmelCase   :					Optional[int]        =    convert_dalle_checkpoint(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							save_checkpoint=SCREAMING_SNAKE_CASE				)

     if os.path.exists(SCREAMING_SNAKE_CASE				):
          _lowerCAmelCase   :					Dict        =    torch.load(SCREAMING_SNAKE_CASE		,							map_location="cpu"				)
     else:
          _lowerCAmelCase   :					Tuple        =    torch.hub.load_state_dict_from_url(SCREAMING_SNAKE_CASE		,							map_location="cpu"				)

     _lowerCAmelCase   :					Optional[int]        =    upgrade_state_dict(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE				)
     hf_model.load_state_dict(SCREAMING_SNAKE_CASE				)
     _lowerCAmelCase   :					Optional[Any]        =    hf_model.state_dict()
     _lowerCAmelCase   :					Optional[Any]        =    count_parameters(SCREAMING_SNAKE_CASE				)
     _lowerCAmelCase   :					Optional[Any]        =    count_parameters(SCREAMING_SNAKE_CASE				) + count_parameters(SCREAMING_SNAKE_CASE				)

     assert torch.allclose(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							atol=1e-3				)

     hf_model.save_pretrained(SCREAMING_SNAKE_CASE				)


if __name__ == "__main__":
  __UpperCAmelCase    				=     argparse.ArgumentParser()
  parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
  parser.add_argument('--checkpoint_path', default=None, type=str, help='Path to flava checkpoint')
  parser.add_argument('--codebook_path', default=None, type=str, help='Path to flava codebook checkpoint')
  parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
  __UpperCAmelCase    				=     parser.parse_args()

  convert_flava_checkpoint(args.checkpoint_path, args.codebook_path, args.pytorch_dump_folder_path, args.config_path)






