





import argparse

import torch
from datasets import load_dataset
from donut import DonutModel

from transformers import (
    DonutImageProcessor,
    DonutProcessor,
    DonutSwinConfig,
    DonutSwinModel,
    MBartConfig,
    MBartForCausalLM,
    VisionEncoderDecoderModel,
    XLMRobertaTokenizerFast,
)




def    A_(			A   ):
	UpperCAmelCase_								=     model.config

	UpperCAmelCase_								=     DonutSwinConfig(
	    image_size=original_config.input_size				,      patch_size=4				,      depths=original_config.encoder_layer				,      num_heads=[4, 8, 16, 32]				,      window_size=original_config.window_size				,      embed_dim=128				,      )
	UpperCAmelCase_								=     MBartConfig(
	    is_decoder=A				,      is_encoder_decoder=A				,      add_cross_attention=A				,      decoder_layers=original_config.decoder_layer				,      max_position_embeddings=original_config.max_position_embeddings				,      vocab_size=len(
	        model.decoder.tokenizer   )				,      scale_embedding=A				,      add_final_layer_norm=A				,      )

	return encoder_config, decoder_config




def    A_(			A   ):







	if "encoder.model" in name:
		UpperCAmelCase_								=     name.replace("""encoder.model"""				,      """encoder"""   )
	if "decoder.model" in name:
		UpperCAmelCase_								=     name.replace("""decoder.model"""				,      """decoder"""   )
	if "patch_embed.proj" in name:
		UpperCAmelCase_								=     name.replace("""patch_embed.proj"""				,      """embeddings.patch_embeddings.projection"""   )
	if "patch_embed.norm" in name:
		UpperCAmelCase_								=     name.replace("""patch_embed.norm"""				,      """embeddings.norm"""   )
	if name.startswith("""encoder"""   ):
		if "layers" in name:
			UpperCAmelCase_								=     """encoder.""" + name
		if "attn.proj" in name:
			UpperCAmelCase_								=     name.replace("""attn.proj"""				,      """attention.output.dense"""   )
		if "attn" in name and "mask" not in name:
			UpperCAmelCase_								=     name.replace("""attn"""				,      """attention.self"""   )
		if "norm1" in name:
			UpperCAmelCase_								=     name.replace("""norm1"""				,      """layernorm_before"""   )
		if "norm2" in name:
			UpperCAmelCase_								=     name.replace("""norm2"""				,      """layernorm_after"""   )
		if "mlp.fc1" in name:
			UpperCAmelCase_								=     name.replace("""mlp.fc1"""				,      """intermediate.dense"""   )
		if "mlp.fc2" in name:
			UpperCAmelCase_								=     name.replace("""mlp.fc2"""				,      """output.dense"""   )

		if name == "encoder.norm.weight":
			UpperCAmelCase_								=     """encoder.layernorm.weight"""
		if name == "encoder.norm.bias":
			UpperCAmelCase_								=     """encoder.layernorm.bias"""

	return name




def    A_(			A				,      A   ):







	for key in orig_state_dict.copy().keys():
		UpperCAmelCase_								=     orig_state_dict.pop(A   )

		if "qkv" in key:
			UpperCAmelCase_								=     key.split("""."""   )
			UpperCAmelCase_								=     int(key_split[3]   )
			UpperCAmelCase_								=     int(key_split[5]   )
			UpperCAmelCase_								=     model.encoder.encoder.layers[layer_num].blocks[block_num].attention.self.all_head_size

			if "weight" in key:
				UpperCAmelCase_								=     val[:dim, :]
				UpperCAmelCase_								=     val[dim : dim * 2, :]
				UpperCAmelCase_								=     val[-dim:, :]
			else:
				UpperCAmelCase_								=     val[:dim]
				UpperCAmelCase_								=     val[dim : dim * 2]
				UpperCAmelCase_								=     val[-dim:]
		elif "attn_mask" in key or key in ["encoder.model.norm.weight", "encoder.model.norm.bias"]:
			# HuggingFace implementation doesn't use attn_mask buffer
			# and model doesn't use final LayerNorms for the encoder
			pass
		else:
			UpperCAmelCase_								=     val

	return orig_state_dict




def    A_(			A				,      A=None				,      A=False   ):
	UpperCAmelCase_								=     DonutModel.from_pretrained(A   ).eval()

	# load HuggingFace model
	UpperCAmelCase_								=     get_configs(A   )
	UpperCAmelCase_								=     DonutSwinModel(A   )
	UpperCAmelCase_								=     MBartForCausalLM(A   )
	UpperCAmelCase_								=     VisionEncoderDecoderModel(encoder=A				,      decoder=A   )
	model.eval()

	UpperCAmelCase_								=     original_model.state_dict()
	UpperCAmelCase_								=     convert_state_dict(A				,      A   )
	model.load_state_dict(A   )

	# verify results on scanned document
	UpperCAmelCase_								=     load_dataset("""hf-internal-testing/example-documents"""   )
	UpperCAmelCase_								=     dataset["""test"""][0]["""image"""].convert("""RGB"""   )

	UpperCAmelCase_								=     XLMRobertaTokenizerFast.from_pretrained(A				,      from_slow=A   )
	UpperCAmelCase_								=     DonutImageProcessor(
	    do_align_long_axis=original_model.config.align_long_axis				,      size=original_model.config.input_size[::-1]   )
	UpperCAmelCase_								=     DonutProcessor(A				,      A   )
	UpperCAmelCase_								=     processor(A				,      return_tensors="""pt"""   ).pixel_values

	if model_name == "naver-clova-ix/donut-base-finetuned-docvqa":
		UpperCAmelCase_								=     """<s_docvqa><s_question>{user_input}</s_question><s_answer>"""
		UpperCAmelCase_								=     """When is the coffee break?"""
		UpperCAmelCase_								=     task_prompt.replace("""{user_input}"""				,      A   )
	elif model_name == "naver-clova-ix/donut-base-finetuned-rvlcdip":
		UpperCAmelCase_								=     """<s_rvlcdip>"""
	elif model_name in [
	    "naver-clova-ix/donut-base-finetuned-cord-v1",
	    "naver-clova-ix/donut-base-finetuned-cord-v1-2560",
	]:
		UpperCAmelCase_								=     """<s_cord>"""
	elif model_name == "naver-clova-ix/donut-base-finetuned-cord-v2":
		UpperCAmelCase_								=     """s_cord-v2>"""
	elif model_name == "naver-clova-ix/donut-base-finetuned-zhtrainticket":
		UpperCAmelCase_								=     """<s_zhtrainticket>"""
	elif model_name in ["naver-clova-ix/donut-proto", "naver-clova-ix/donut-base"]:
		# use a random prompt
		UpperCAmelCase_								=     """hello world"""
	else:
		raise ValueError("""Model name not supported"""   )
	UpperCAmelCase_								=     original_model.decoder.tokenizer(A				,      add_special_tokens=A				,      return_tensors="""pt"""   )[
	    """input_ids"""
	]

	UpperCAmelCase_								=     original_model.encoder.model.patch_embed(A   )
	UpperCAmelCase_								=     model.encoder.embeddings(A   )
	assert torch.allclose(A				,      A				,      atol=1E-3   )

	# verify encoder hidden states
	UpperCAmelCase_								=     original_model.encoder(A   )
	UpperCAmelCase_								=     model.encoder(A   ).last_hidden_state
	assert torch.allclose(A				,      A				,      atol=1E-2   )

	# verify decoder hidden states
	UpperCAmelCase_								=     original_model(A				,      A				,      A   ).logits
	UpperCAmelCase_								=     model(A				,      decoder_input_ids=A   ).logits
	assert torch.allclose(A				,      A				,      atol=1E-3   )
	print("""Looks ok!"""   )

	if pytorch_dump_folder_path is not None:
		print(f"""Saving model and processor to {pytorch_dump_folder_path}"""   )
		model.save_pretrained(A   )
		processor.save_pretrained(A   )

	if push_to_hub:
		model.push_to_hub("""nielsr/""" + model_name.split("""/"""   )[-1]				,      commit_message="""Update model"""   )
		processor.push_to_hub("""nielsr/""" + model_name.split("""/"""   )[-1]				,      commit_message="""Update model"""   )


if __name__ == "__main__":
				UpperCamelCase__   : Union[str, Any]					  = argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    """--model_name""",
				    default="""naver-clova-ix/donut-base-finetuned-docvqa""",
				    required=False,
				    type=str,
				    help="""Name of the original model you\'d like to convert.""",
				)
				parser.add_argument(
				    """--pytorch_dump_folder_path""",
				    default=None,
				    required=False,
				    type=str,
				    help="""Path to the output PyTorch model directory.""",
				)
				parser.add_argument(
				    """--push_to_hub""",
				    action="""store_true""",
				    help="""Whether or not to push the converted model and processor to the 🤗 hub.""",
				)

				UpperCamelCase__   : Tuple					  = parser.parse_args()
				convert_donut_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)









import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def    A_(			A				,      A   ):
	assert isinstance(A				,      A   )
	assert dataset.num_rows == 4
	assert dataset.num_columns == 1
	assert dataset.column_names == ["text"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory"""				,      [False, True]   )
def    A_(			A				,      A				,      A   ):
	UpperCAmelCase_								=     tmp_path / """cache"""
	UpperCAmelCase_								=     {"""text""": """string"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		UpperCAmelCase_								=     TextDatasetReader(A				,      cache_dir=A				,      keep_in_memory=A   ).read()
	_check_text_dataset(A				,      A   )




@pytest.mark.parametrize(
    """features"""				,      [
        None,
        {"""text""": """string"""},
        {"""text""": """int32"""},
        {"""text""": """float32"""},
    ]				,      )
def    A_(			A				,      A				,      A   ):
	UpperCAmelCase_								=     tmp_path / """cache"""
	UpperCAmelCase_								=     {"""text""": """string"""}
	UpperCAmelCase_								=     features.copy() if features else default_expected_features
	UpperCAmelCase_								=     (
	    Features({feature: Value(A   ) for feature, dtype in features.items()}   ) if features is not None else None
	)
	UpperCAmelCase_								=     TextDatasetReader(A				,      features=A				,      cache_dir=A   ).read()
	_check_text_dataset(A				,      A   )




@pytest.mark.parametrize("""split"""				,      [None, NamedSplit("""train"""   ), """train""", """test"""]   )
def    A_(			A				,      A				,      A   ):
	UpperCAmelCase_								=     tmp_path / """cache"""
	UpperCAmelCase_								=     {"""text""": """string"""}
	UpperCAmelCase_								=     TextDatasetReader(A				,      cache_dir=A				,      split=A   ).read()
	_check_text_dataset(A				,      A   )
	assert dataset.split == split if split else "train"




@pytest.mark.parametrize("""path_type"""				,      [str, list]   )
def    A_(			A				,      A				,      A   ):
	if issubclass(A				,      A   ):
		UpperCAmelCase_								=     text_path
	elif issubclass(A				,      A   ):
		UpperCAmelCase_								=     [text_path]
	UpperCAmelCase_								=     tmp_path / """cache"""
	UpperCAmelCase_								=     {"""text""": """string"""}
	UpperCAmelCase_								=     TextDatasetReader(A				,      cache_dir=A   ).read()
	_check_text_dataset(A				,      A   )




def    A_(			A				,      A				,      A=("train",)   ):
	assert isinstance(A				,      A   )
	for split in splits:
		UpperCAmelCase_								=     dataset_dict[split]
		assert dataset.num_rows == 4
		assert dataset.num_columns == 1
		assert dataset.column_names == ["text"]
		for feature, expected_dtype in expected_features.items():
			assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory"""				,      [False, True]   )
def    A_(			A				,      A				,      A   ):
	UpperCAmelCase_								=     tmp_path / """cache"""
	UpperCAmelCase_								=     {"""text""": """string"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		UpperCAmelCase_								=     TextDatasetReader({"""train""": text_path}				,      cache_dir=A				,      keep_in_memory=A   ).read()
	_check_text_datasetdict(A				,      A   )




@pytest.mark.parametrize(
    """features"""				,      [
        None,
        {"""text""": """string"""},
        {"""text""": """int32"""},
        {"""text""": """float32"""},
    ]				,      )
def    A_(			A				,      A				,      A   ):
	UpperCAmelCase_								=     tmp_path / """cache"""
	# CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
	UpperCAmelCase_								=     {"""text""": """string"""}
	UpperCAmelCase_								=     features.copy() if features else default_expected_features
	UpperCAmelCase_								=     (
	    Features({feature: Value(A   ) for feature, dtype in features.items()}   ) if features is not None else None
	)
	UpperCAmelCase_								=     TextDatasetReader({"""train""": text_path}				,      features=A				,      cache_dir=A   ).read()
	_check_text_datasetdict(A				,      A   )




@pytest.mark.parametrize("""split"""				,      [None, NamedSplit("""train"""   ), """train""", """test"""]   )
def    A_(			A				,      A				,      A   ):
	if split:
		UpperCAmelCase_								=     {split: text_path}
	else:
		UpperCAmelCase_								=     """train"""
		UpperCAmelCase_								=     {"""train""": text_path, """test""": text_path}
	UpperCAmelCase_								=     tmp_path / """cache"""
	UpperCAmelCase_								=     {"""text""": """string"""}
	UpperCAmelCase_								=     TextDatasetReader(A				,      cache_dir=A   ).read()
	_check_text_datasetdict(A				,      A				,      splits=list(path.keys()   )   )
	assert all(dataset[split].split == split for split in path.keys()   )


