'''simple docstring'''

import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
lowercase_     		= logging.get_logger(__name__)



def 							UpperCamelCase__			(			a__			,      a__=False			,      a__=False			,      a__=False      ):


						'''simple docstring'''






						_lowerCAmelCase										=[]
						for i in range(config.num_hidden_layers      ):
												# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
												rename_keys.append((F'''transformer.blocks.{i}.norm1.weight''', F'''vilt.encoder.layer.{i}.layernorm_before.weight''')      )
												rename_keys.append((F'''transformer.blocks.{i}.norm1.bias''', F'''vilt.encoder.layer.{i}.layernorm_before.bias''')      )
												rename_keys.append(
												    (F'''transformer.blocks.{i}.attn.proj.weight''', F'''vilt.encoder.layer.{i}.attention.output.dense.weight''')      )
												rename_keys.append(
												    (F'''transformer.blocks.{i}.attn.proj.bias''', F'''vilt.encoder.layer.{i}.attention.output.dense.bias''')      )
												rename_keys.append((F'''transformer.blocks.{i}.norm2.weight''', F'''vilt.encoder.layer.{i}.layernorm_after.weight''')      )
												rename_keys.append((F'''transformer.blocks.{i}.norm2.bias''', F'''vilt.encoder.layer.{i}.layernorm_after.bias''')      )
												rename_keys.append(
												    (F'''transformer.blocks.{i}.mlp.fc1.weight''', F'''vilt.encoder.layer.{i}.intermediate.dense.weight''')      )
												rename_keys.append((F'''transformer.blocks.{i}.mlp.fc1.bias''', F'''vilt.encoder.layer.{i}.intermediate.dense.bias''')      )
												rename_keys.append((F'''transformer.blocks.{i}.mlp.fc2.weight''', F'''vilt.encoder.layer.{i}.output.dense.weight''')      )
												rename_keys.append((F'''transformer.blocks.{i}.mlp.fc2.bias''', F'''vilt.encoder.layer.{i}.output.dense.bias''')      )

						# embeddings
						rename_keys.extend(
						    [
						        # text embeddings
						        ('text_embeddings.word_embeddings.weight', 'vilt.embeddings.text_embeddings.word_embeddings.weight'),
						        (
						            'text_embeddings.position_embeddings.weight',
						            'vilt.embeddings.text_embeddings.position_embeddings.weight',
						        ),
						        ('text_embeddings.position_ids', 'vilt.embeddings.text_embeddings.position_ids'),
						        (
						            'text_embeddings.token_type_embeddings.weight',
						            'vilt.embeddings.text_embeddings.token_type_embeddings.weight',
						        ),
						        ('text_embeddings.LayerNorm.weight', 'vilt.embeddings.text_embeddings.LayerNorm.weight'),
						        ('text_embeddings.LayerNorm.bias', 'vilt.embeddings.text_embeddings.LayerNorm.bias'),
						        # patch embeddings
						        ('transformer.cls_token', 'vilt.embeddings.cls_token'),
						        ('transformer.patch_embed.proj.weight', 'vilt.embeddings.patch_embeddings.projection.weight'),
						        ('transformer.patch_embed.proj.bias', 'vilt.embeddings.patch_embeddings.projection.bias'),
						        ('transformer.pos_embed', 'vilt.embeddings.position_embeddings'),
						        # token type embeddings
						        ('token_type_embeddings.weight', 'vilt.embeddings.token_type_embeddings.weight'),
						    ]      )

						# final layernorm + pooler
						rename_keys.extend(
						    [
						        ('transformer.norm.weight', 'vilt.layernorm.weight'),
						        ('transformer.norm.bias', 'vilt.layernorm.bias'),
						        ('pooler.dense.weight', 'vilt.pooler.dense.weight'),
						        ('pooler.dense.bias', 'vilt.pooler.dense.bias'),
						    ]      )

						# classifier head(s)
						if vqa_model:
												# classification head
												rename_keys.extend(
												    [
												        ('vqa_classifier.0.weight', 'classifier.0.weight'),
												        ('vqa_classifier.0.bias', 'classifier.0.bias'),
												        ('vqa_classifier.1.weight', 'classifier.1.weight'),
												        ('vqa_classifier.1.bias', 'classifier.1.bias'),
												        ('vqa_classifier.3.weight', 'classifier.3.weight'),
												        ('vqa_classifier.3.bias', 'classifier.3.bias'),
												    ]      )
						elif nlvr_model:
												# classification head
												rename_keys.extend(
												    [
												        ('nlvr2_classifier.0.weight', 'classifier.0.weight'),
												        ('nlvr2_classifier.0.bias', 'classifier.0.bias'),
												        ('nlvr2_classifier.1.weight', 'classifier.1.weight'),
												        ('nlvr2_classifier.1.bias', 'classifier.1.bias'),
												        ('nlvr2_classifier.3.weight', 'classifier.3.weight'),
												        ('nlvr2_classifier.3.bias', 'classifier.3.bias'),
												    ]      )
						else:
												pass

						return rename_keys



def 							UpperCamelCase__			(			a__			,      a__      ):


						'''simple docstring'''






						for i in range(config.num_hidden_layers      ):
												_lowerCAmelCase										="""vilt."""
												# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
												_lowerCAmelCase										=state_dict.pop(F'''transformer.blocks.{i}.attn.qkv.weight'''      )
												_lowerCAmelCase										=state_dict.pop(F'''transformer.blocks.{i}.attn.qkv.bias'''      )
												# next, add query, keys and values (in that order) to the state dict
												_lowerCAmelCase										=in_proj_weight[
												    : config.hidden_size, :
												]
												_lowerCAmelCase										=in_proj_bias[: config.hidden_size]
												_lowerCAmelCase										=in_proj_weight[
												    config.hidden_size : config.hidden_size * 2, :
												]
												_lowerCAmelCase										=in_proj_bias[
												    config.hidden_size : config.hidden_size * 2
												]
												_lowerCAmelCase										=in_proj_weight[
												    -config.hidden_size :, :
												]
												_lowerCAmelCase										=in_proj_bias[-config.hidden_size :]



def 							UpperCamelCase__			(			a__      ):


						'''simple docstring'''






						_lowerCAmelCase										=["""head.weight""", """head.bias"""]
						for k in ignore_keys:
												state_dict.pop(UpperCAmelCase__			,      UpperCAmelCase__      )



def 							UpperCamelCase__			(			a__			,      a__			,      a__      ):


						'''simple docstring'''






						_lowerCAmelCase										=dct.pop(UpperCAmelCase__      )
						_lowerCAmelCase										=val



@torch.no_grad()
def 							UpperCamelCase__			(			a__			,      a__      ):


						'''simple docstring'''






						_lowerCAmelCase										=ViltConfig(image_size=3_8_4			,      patch_size=3_2			,      tie_word_embeddings=UpperCAmelCase__      )
						_lowerCAmelCase										=False
						_lowerCAmelCase										=False
						_lowerCAmelCase										=False
						_lowerCAmelCase										=False
						if "vqa" in checkpoint_url:
												_lowerCAmelCase										=True
												_lowerCAmelCase										=3_1_2_9
												_lowerCAmelCase										="""huggingface/label-files"""
												_lowerCAmelCase										="""vqa2-id2label.json"""
												_lowerCAmelCase										=json.load(open(hf_hub_download(UpperCAmelCase__			,      UpperCAmelCase__			,      repo_type='dataset'      )			,      'r'      )      )
												_lowerCAmelCase										={int(UpperCAmelCase__      ): v for k, v in idalabel.items()}
												_lowerCAmelCase										=idalabel
												_lowerCAmelCase										={v: k for k, v in idalabel.items()}
												_lowerCAmelCase										=ViltForQuestionAnswering(UpperCAmelCase__      )
						elif "nlvr" in checkpoint_url:
												_lowerCAmelCase										=True
												_lowerCAmelCase										=2
												_lowerCAmelCase										={0: """False""", 1: """True"""}
												_lowerCAmelCase										={v: k for k, v in config.idalabel.items()}
												_lowerCAmelCase										=3
												_lowerCAmelCase										=ViltForImagesAndTextClassification(UpperCAmelCase__      )
						elif "irtr" in checkpoint_url:
												_lowerCAmelCase										=True
												_lowerCAmelCase										=ViltForImageAndTextRetrieval(UpperCAmelCase__      )
						elif "mlm_itm" in checkpoint_url:
												_lowerCAmelCase										=True
												_lowerCAmelCase										=ViltForMaskedLM(UpperCAmelCase__      )
						else:
												raise ValueError('Unknown model type'      )

						# load state_dict of original model, remove and rename some keys
						_lowerCAmelCase										=torch.hub.load_state_dict_from_url(UpperCAmelCase__			,      map_location='cpu'      )["""state_dict"""]
						_lowerCAmelCase										=create_rename_keys(UpperCAmelCase__			,      UpperCAmelCase__			,      UpperCAmelCase__			,      UpperCAmelCase__      )
						for src, dest in rename_keys:
												rename_key(UpperCAmelCase__			,      UpperCAmelCase__			,      UpperCAmelCase__      )
						read_in_q_k_v(UpperCAmelCase__			,      UpperCAmelCase__      )
						if mlm_model or irtr_model:
												_lowerCAmelCase										=["""itm_score.fc.weight""", """itm_score.fc.bias"""]
												for k in ignore_keys:
																		state_dict.pop(UpperCAmelCase__			,      UpperCAmelCase__      )

    # load state dict into HuggingFace model
						model.eval()
						if mlm_model:
												_lowerCAmelCase										=model.load_state_dict(UpperCAmelCase__			,      strict=UpperCAmelCase__      )
												assert missing_keys == ["mlm_score.decoder.bias"]
						else:
												model.load_state_dict(UpperCAmelCase__      )

						# Define processor
						_lowerCAmelCase										=ViltImageProcessor(size=3_8_4      )
						_lowerCAmelCase										=BertTokenizer.from_pretrained('bert-base-uncased'      )
						_lowerCAmelCase										=ViltProcessor(UpperCAmelCase__			,      UpperCAmelCase__      )

						# Forward pass on example inputs (image + text)
						if nlvr_model:
												_lowerCAmelCase										=Image.open(requests.get('https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'			,      stream=UpperCAmelCase__      ).raw      )
												_lowerCAmelCase										=Image.open(requests.get('https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'			,      stream=UpperCAmelCase__      ).raw      )
												_lowerCAmelCase										=(
												    """The left image contains twice the number of dogs as the right image, and at least two dogs in total are"""
												    """ standing."""
												)
												_lowerCAmelCase										=processor(UpperCAmelCase__			,      UpperCAmelCase__			,      return_tensors='pt'      )
												_lowerCAmelCase										=processor(UpperCAmelCase__			,      UpperCAmelCase__			,      return_tensors='pt'      )
												_lowerCAmelCase										=model(
												    input_ids=encoding_a.input_ids			,      pixel_values=encoding_a.pixel_values			,      pixel_values_a=encoding_a.pixel_values			,      )
						else:
												_lowerCAmelCase										=Image.open(requests.get('http://images.cocodataset.org/val2017/000000039769.jpg'			,      stream=UpperCAmelCase__      ).raw      )
												if mlm_model:
																		_lowerCAmelCase										="""a bunch of [MASK] laying on a [MASK]."""
												else:
																		_lowerCAmelCase										="""How many cats are there?"""
												_lowerCAmelCase										=processor(UpperCAmelCase__			,      UpperCAmelCase__			,      return_tensors='pt'      )
												_lowerCAmelCase										=model(**UpperCAmelCase__      )

						# Verify outputs
						if mlm_model:
												_lowerCAmelCase										=torch.Size([1, 1_1, 3_0_5_2_2]      )
												_lowerCAmelCase										=torch.tensor([-12.5_061, -12.5_123, -12.5_174]      )
												assert outputs.logits.shape == expected_shape
												assert torch.allclose(outputs.logits[0, 0, :3]			,      UpperCAmelCase__			,      atol=1E-4      )

												# verify masked token prediction equals "cats"
												_lowerCAmelCase										=outputs.logits[0, 4, :].argmax(-1      ).item()
												assert tokenizer.decode([predicted_id]      ) == "cats"
						elif vqa_model:
												_lowerCAmelCase										=torch.Size([1, 3_1_2_9]      )
												_lowerCAmelCase										=torch.tensor([-15.9_495, -18.1_472, -10.3_041]      )
												assert torch.allclose(outputs.logits[0, :3]			,      UpperCAmelCase__			,      atol=1E-4      )
												assert outputs.logits.shape == expected_shape
												assert torch.allclose(outputs.logits[0, 0, :3]			,      UpperCAmelCase__			,      atol=1E-4      )

												# verify vqa prediction equals "2"
												_lowerCAmelCase										=outputs.logits.argmax(-1      ).item()
												assert model.config.idalabel[predicted_idx] == "2"
						elif nlvr_model:
												_lowerCAmelCase										=torch.Size([1, 2]      )
												_lowerCAmelCase										=torch.tensor([-2.8_721, 2.1_291]      )
												assert torch.allclose(outputs.logits[0, :3]			,      UpperCAmelCase__			,      atol=1E-4      )
												assert outputs.logits.shape == expected_shape

						Path(UpperCAmelCase__      ).mkdir(exist_ok=UpperCAmelCase__      )
						print(F'''Saving model and processor to {pytorch_dump_folder_path}'''      )
						model.save_pretrained(UpperCAmelCase__      )
						processor.save_pretrained(UpperCAmelCase__      )


if __name__ == "__main__":
							lowercase_     		= argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '''--checkpoint_url''',
							    default='''https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt''',
							    type=str,
							    help='''URL of the checkpoint you\'d like to convert.''',
							)
							parser.add_argument(
							    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
							)

							lowercase_     		= parser.parse_args()
							convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)






'''simple docstring'''

import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
							raise RuntimeError('''At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training''')

# TF training parameters
lowercase_     		= False
lowercase_     		= False



def 							UpperCamelCase__			(			a__      ):


						'''simple docstring'''






						return TrainCommand(a__      )


class   SCREAMING_SNAKE_CASE			(						__lowercase):







		"""simple docstring"""






		@staticmethod
		def 		UpperCamelCase__       (   __A				) ->     Tuple:
								_lowerCAmelCase										=parser.add_parser('train'		,						help='CLI tool to train a model on a task.'				)

								train_parser.add_argument(
								    '--train_data'		,						type=__A		,						required=__A		,						help='path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences.'		,						)
								train_parser.add_argument(
								    '--column_label'		,						type=__A		,						default=0		,						help='Column of the dataset csv file with example labels.'				)
								train_parser.add_argument(
								    '--column_text'		,						type=__A		,						default=1		,						help='Column of the dataset csv file with example texts.'				)
								train_parser.add_argument(
								    '--column_id'		,						type=__A		,						default=2		,						help='Column of the dataset csv file with example ids.'				)
								train_parser.add_argument(
								    '--skip_first_row'		,						action='store_true'		,						help='Skip the first row of the csv file (headers).'				)

								train_parser.add_argument('--validation_data'		,						type=__A		,						default=''		,						help='path to validation dataset.'				)
								train_parser.add_argument(
								    '--validation_split'		,						type=__A		,						default=0.1		,						help='if validation dataset is not provided, fraction of train dataset to use as validation dataset.'		,						)

								train_parser.add_argument('--output'		,						type=__A		,						default='./'		,						help='path to saved the trained model.'				)

								train_parser.add_argument(
								    '--task'		,						type=__A		,						default='text_classification'		,						help='Task to train the model on.'				)
								train_parser.add_argument(
								    '--model'		,						type=__A		,						default='bert-base-uncased'		,						help='Model\'s name or path to stored model.'				)
								train_parser.add_argument('--train_batch_size'		,						type=__A		,						default=32		,						help='Batch size for training.'				)
								train_parser.add_argument('--valid_batch_size'		,						type=__A		,						default=64		,						help='Batch size for validation.'				)
								train_parser.add_argument('--learning_rate'		,						type=__A		,						default=3E-5		,						help='Learning rate.'				)
								train_parser.add_argument('--adam_epsilon'		,						type=__A		,						default=1E-08		,						help='Epsilon for Adam optimizer.'				)
								train_parser.set_defaults(func=__A				)






		def __init__(   self		,						__A				) ->     List[str]:
								_lowerCAmelCase										=logging.get_logger('transformers-cli/training'				)

								_lowerCAmelCase										='tf' if is_tf_available() else 'torch'

								os.makedirs(args.output		,						exist_ok=__A				)
								_lowerCAmelCase										=args.output

								_lowerCAmelCase										=args.column_label
								_lowerCAmelCase										=args.column_text
								_lowerCAmelCase										=args.column_id

								self.logger.info(F'''Loading {args.task} pipeline for {args.model}'''				)
								if args.task == "text_classification":
														_lowerCAmelCase										=TextClassificationPipeline.from_pretrained(args.model				)
								elif args.task == "token_classification":
														raise NotImplementedError
								elif args.task == "question_answering":
														raise NotImplementedError

								self.logger.info(F'''Loading dataset from {args.train_data}'''				)
								_lowerCAmelCase										=Processor.create_from_csv(
								    args.train_data		,						column_label=args.column_label		,						column_text=args.column_text		,						column_id=args.column_id		,						skip_first_row=args.skip_first_row		,						)
								_lowerCAmelCase										=None
								if args.validation_data:
														self.logger.info(F'''Loading validation dataset from {args.validation_data}'''				)
														_lowerCAmelCase										=Processor.create_from_csv(
														    args.validation_data		,						column_label=args.column_label		,						column_text=args.column_text		,						column_id=args.column_id		,						skip_first_row=args.skip_first_row		,						)

								_lowerCAmelCase										=args.validation_split
								_lowerCAmelCase										=args.train_batch_size
								_lowerCAmelCase										=args.valid_batch_size
								_lowerCAmelCase										=args.learning_rate
								_lowerCAmelCase										=args.adam_epsilon






		def 		UpperCamelCase__       (   self				) ->     List[str]:
								if self.framework == "tf":
														return self.run_tf()
								return self.run_torch()






		def 		UpperCamelCase__       (   self				) ->     Union[str, Any]:
								raise NotImplementedError






		def 		UpperCamelCase__       (   self				) ->     List[Any]:
								self.pipeline.fit(
								    self.train_dataset		,						validation_data=self.valid_dataset		,						validation_split=self.validation_split		,						learning_rate=self.learning_rate		,						adam_epsilon=self.adam_epsilon		,						train_batch_size=self.train_batch_size		,						valid_batch_size=self.valid_batch_size		,						)

								# Save trained pipeline
								self.pipeline.save_pretrained(self.output				)





