



import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    LiltForQuestionAnswering,
							    LiltForSequenceClassification,
							    LiltForTokenClassification,
							    LiltModel,
							)
							from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST
class 				_snake_case			:




				def __init__(						self					:						Optional[int],					__lowercase					:						str,					__lowercase					:						str=13,					__lowercase					:						List[Any]=7,					__lowercase					:						Any=True,					__lowercase					:						Optional[int]=True,					__lowercase					:						int=True,					__lowercase					:						Any=True,					__lowercase					:						str=99,					__lowercase					:						Optional[Any]=24,					__lowercase					:						int=2,					__lowercase					:						Optional[Any]=6,					__lowercase					:						Optional[int]=37,					__lowercase					:						int="gelu",					__lowercase					:						Dict=0.1,					__lowercase					:						str=0.1,					__lowercase					:						str=512,					__lowercase					:						int=16,					__lowercase					:						Union[str, Any]=2,					__lowercase					:						int=0.02,					__lowercase					:						str=3,					__lowercase					:						Optional[Any]=None,					__lowercase					:						Optional[Any]=1000,					):
							lowercase__					  =				parent
							lowercase__					  =				batch_size
							lowercase__					  =				seq_length
							lowercase__					  =				is_training
							lowercase__					  =				use_input_mask
							lowercase__					  =				use_token_type_ids
							lowercase__					  =				use_labels
							lowercase__					  =				vocab_size
							lowercase__					  =				hidden_size
							lowercase__					  =				num_hidden_layers
							lowercase__					  =				num_attention_heads
							lowercase__					  =				intermediate_size
							lowercase__					  =				hidden_act
							lowercase__					  =				hidden_dropout_prob
							lowercase__					  =				attention_probs_dropout_prob
							lowercase__					  =				max_position_embeddings
							lowercase__					  =				type_vocab_size
							lowercase__					  =				type_sequence_label_size
							lowercase__					  =				initializer_range
							lowercase__					  =				num_labels
							lowercase__					  =				scope
							lowercase__					  =				range_bbox




				def 							A__      (						self					:						List[str]						):
							lowercase__					  =				ids_tensor([self.batch_size, self.seq_length],					self.vocab_size						)

							lowercase__					  =				ids_tensor([self.batch_size, self.seq_length, 4],					self.range_bbox						)
							# Ensure that bbox is legal
							for i in range(bbox.shape[0]						):
										for j in range(bbox.shape[1]						):
													if bbox[i, j, 3] < bbox[i, j, 1]:
																lowercase__					  =				bbox[i, j, 3]
																lowercase__					  =				bbox[i, j, 1]
																lowercase__					  =				t
													if bbox[i, j, 2] < bbox[i, j, 0]:
																lowercase__					  =				bbox[i, j, 2]
																lowercase__					  =				bbox[i, j, 0]
																lowercase__					  =				t

							lowercase__					  =				None
							if self.use_input_mask:
										lowercase__					  =				ids_tensor([self.batch_size, self.seq_length],					vocab_size=2						)

							lowercase__					  =				None
							if self.use_token_type_ids:
										lowercase__					  =				ids_tensor([self.batch_size, self.seq_length],					self.type_vocab_size						)

							lowercase__					  =				None
							lowercase__					  =				None
							if self.use_labels:
										lowercase__					  =				ids_tensor([self.batch_size],					self.type_sequence_label_size						)
										lowercase__					  =				ids_tensor([self.batch_size, self.seq_length],					self.num_labels						)

							lowercase__					  =				self.get_config()

							return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels




				def 							A__      (						self					:						Dict						):
							return LiltConfig(
							    vocab_size=self.vocab_size,					hidden_size=self.hidden_size,					num_hidden_layers=self.num_hidden_layers,					num_attention_heads=self.num_attention_heads,					intermediate_size=self.intermediate_size,					hidden_act=self.hidden_act,					hidden_dropout_prob=self.hidden_dropout_prob,					attention_probs_dropout_prob=self.attention_probs_dropout_prob,					max_position_embeddings=self.max_position_embeddings,					type_vocab_size=self.type_vocab_size,					initializer_range=self.initializer_range,					)




				def 							A__      (						self					:						Tuple,					__lowercase					:						Optional[int],					__lowercase					:						List[Any],					__lowercase					:						int,					__lowercase					:						int,					__lowercase					:						Union[str, Any],					__lowercase					:						Any,					__lowercase					:						str,					):
							lowercase__					  =				LiltModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							lowercase__					  =				model(__lowercase,					bbox=__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase						)
							lowercase__					  =				model(__lowercase,					bbox=__lowercase,					token_type_ids=__lowercase						)
							lowercase__					  =				model(__lowercase,					bbox=__lowercase						)

							self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size)						)
							self.parent.assertEqual(result.pooler_output.shape,					(self.batch_size, self.hidden_size)						)




				def 							A__      (						self					:						Optional[int],					__lowercase					:						Dict,					__lowercase					:						Optional[int],					__lowercase					:						List[str],					__lowercase					:						Tuple,					__lowercase					:						List[Any],					__lowercase					:						List[str],					__lowercase					:						str,					):
							lowercase__					  =				self.num_labels
							lowercase__					  =				LiltForTokenClassification(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							lowercase__					  =				model(
							    __lowercase,					bbox=__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase,					labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.num_labels)						)




				def 							A__      (						self					:						Dict,					__lowercase					:						List[Any],					__lowercase					:						Union[str, Any],					__lowercase					:						int,					__lowercase					:						Tuple,					__lowercase					:						Optional[int],					__lowercase					:						Any,					__lowercase					:						Dict,					):
							lowercase__					  =				LiltForQuestionAnswering(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							lowercase__					  =				model(
							    __lowercase,					bbox=__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase,					start_positions=__lowercase,					end_positions=__lowercase,					)
							self.parent.assertEqual(result.start_logits.shape,					(self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape,					(self.batch_size, self.seq_length)						)




				def 							A__      (						self					:						Any						):
							lowercase__					  =				self.prepare_config_and_inputs()
							(
							    (
							    lowercase__
							)  ,       (
							    lowercase__
							)  ,       (
							    lowercase__
							)  ,       (
							    lowercase__
							)  ,       (
							    lowercase__
							)  ,       (
							    lowercase__
							)  ,       (
							    lowercase__
							)  ,       
							)					  =				config_and_inputs
							lowercase__					  =				{
							    "input_ids": input_ids,
							    "bbox": bbox,
							    "token_type_ids": token_type_ids,
							    "attention_mask": input_mask,
							}
							return config, inputs_dict
@require_torch
class 				_snake_case			(					lowercase__    ,	lowercase__    ,	lowercase__    ,	unittest.TestCase):
				UpperCamelCase__				:					str     		=(
				    (
				        LiltModel,
				        LiltForSequenceClassification,
				        LiltForTokenClassification,
				        LiltForQuestionAnswering,
				    )
				    if is_torch_available()
				    else ()
				)
				UpperCamelCase__				:					Optional[int]     		=(
				    {
				        """feature-extraction""": LiltModel,
				        """question-answering""": LiltForQuestionAnswering,
				        """text-classification""": LiltForSequenceClassification,
				        """token-classification""": LiltForTokenClassification,
				        """zero-shot""": LiltForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				UpperCamelCase__				:					Any     		=False
				UpperCamelCase__				:					Dict     		=False




				def 							A__      (						self					:						int,					__lowercase					:						Optional[Any],					__lowercase					:						List[str],					__lowercase					:						List[Any],					__lowercase					:						int,					__lowercase					:						str						):
							return True




				def 							A__      (						self					:						List[str]						):
							lowercase__					  =				LiltModelTester(self						)
							lowercase__					  =				ConfigTester(self,					config_class=__lowercase,					hidden_size=37						)




				def 							A__      (						self					:						Optional[int]						):
							self.config_tester.run_common_tests()




				def 							A__      (						self					:						Any						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowercase						)




				def 							A__      (						self					:						str						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
										lowercase__					  =				type
										self.model_tester.create_and_check_model(*__lowercase						)




				def 							A__      (						self					:						List[Any]						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__lowercase						)




				def 							A__      (						self					:						Any						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__lowercase						)




				@slow
				def 							A__      (						self					:						Optional[int]						):
							for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										lowercase__					  =				LiltModel.from_pretrained(__lowercase						)
										self.assertIsNotNone(__lowercase						)
@require_torch
@slow
class 				_snake_case			(					unittest.TestCase):




				def 							A__      (						self					:						Optional[Any]						):
							lowercase__					  =				LiltModel.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base"						).to(__lowercase						)

							lowercase__					  =				torch.tensor([[1, 2]],					device=__lowercase						)
							lowercase__					  =				torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]],					device=__lowercase						)

							# forward pass
							with torch.no_grad():
										lowercase__					  =				model(input_ids=__lowercase,					bbox=__lowercase						)

							lowercase__					  =				torch.Size([1, 2, 768]						)
							lowercase__					  =				torch.tensor(
							    [[-0.0653, 0.0950, -0.0061], [-0.0545, 0.0926, -0.0324]],					device=__lowercase,					)

							self.assertTrue(outputs.last_hidden_state.shape,					__lowercase						)
							self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3],					__lowercase,					atol=1e-3						)						)









import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


lowercase_								=     """<<<<<<< This should probably be modified because it mentions: """

lowercase_								=     """=======
>>>>>>>
"""

lowercase_								=     [
    """TextEncoderConfig""",
    """ByteTextEncoder""",
    """SubwordTextEncoder""",
    """encoder_config""",
    """maybe_build_from_corpus""",
    """manual_dir""",
]

lowercase_								=     [
    # (pattern, replacement)
    # Order is important here for some replacements
    (r"""tfds\.core""", r"""datasets"""),
    (r"""tf\.io\.gfile\.GFile""", r"""open"""),
    (r"""tf\.([\w\d]+)""", r"""datasets.Value('\1')"""),
    (r"""tfds\.features\.Text\(\)""", r"""datasets.Value('string')"""),
    (r"""tfds\.features\.Text\(""", r"""datasets.Value('string'),"""),
    (r"""features\s*=\s*tfds.features.FeaturesDict\(""", r"""features=datasets.Features("""),
    (r"""tfds\.features\.FeaturesDict\(""", r"""dict("""),
    (r"""The TensorFlow Datasets Authors""", r"""The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"""),
    (r"""tfds\.""", r"""datasets."""),
    (r"""dl_manager\.manual_dir""", r"""self.config.data_dir"""),
    (r"""self\.builder_config""", r"""self.config"""),
]



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			return ConvertCommand(args.tfds_path						,  args.datasets_directory  )




class 				_snake_case			(					lowercase__):




				@staticmethod
				def 							A__      (						__lowercase					:						ArgumentParser						):
							lowercase__					  =				parser.add_parser(
							    "convert",					help="Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.",					)
							train_parser.add_argument(
							    "--tfds_path",					type=__lowercase,					required=__lowercase,					help="Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.",					)
							train_parser.add_argument(
							    "--datasets_directory",					type=__lowercase,					required=__lowercase,					help="Path to the HuggingFace Datasets folder."						)
							train_parser.set_defaults(func=__lowercase						)




				def __init__(						self					:						Tuple,					__lowercase					:						str,					__lowercase					:						str,					*__lowercase					:						Tuple						):
							lowercase__					  =				get_logger("datasets-cli/converting"						)

							lowercase__					  =				tfds_path
							lowercase__					  =				datasets_directory




				def 							A__      (						self					:						Any						):
							if os.path.isdir(self._tfds_path						):
										lowercase__					  =				os.path.abspath(self._tfds_path						)
							elif os.path.isfile(self._tfds_path						):
										lowercase__					  =				os.path.dirname(self._tfds_path						)
							else:
										raise ValueError("--tfds_path is neither a directory nor a file. Please check path."						)

							lowercase__					  =				os.path.abspath(self._datasets_directory						)

							self._logger.info(F'''Converting datasets from {abs_tfds_path} to {abs_datasets_path}'''						)

							lowercase__					  =				[]
							lowercase__					  =				[]
							lowercase__					  =				{}

							if os.path.isdir(self._tfds_path						):
										lowercase__					  =				os.listdir(__lowercase						)
							else:
										lowercase__					  =				[os.path.basename(self._tfds_path						)]

							for f_name in file_names:
										self._logger.info(F'''Looking at file {f_name}'''						)
										lowercase__					  =				os.path.join(__lowercase,					__lowercase						)
										lowercase__					  =				os.path.join(__lowercase,					__lowercase						)

										if not os.path.isfile(__lowercase						) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
													self._logger.info("Skipping file"						)
													continue

										with open(__lowercase,					encoding="utf-8"						) as f:
													lowercase__					  =				f.readlines()

										lowercase__					  =				[]
										lowercase__					  =				False
										lowercase__					  =				False
										lowercase__					  =				[]
										for line in lines:
													lowercase__					  =				line

													# Convert imports
													if "import tensorflow.compat.v2 as tf" in out_line:
																continue
													elif "@tfds.core" in out_line:
																continue
													elif "builder=self" in out_line:
																continue
													elif "import tensorflow_datasets.public_api as tfds" in out_line:
																lowercase__					  =				"import datasets\n"
													elif "import tensorflow" in out_line:
																# order is important here
																lowercase__					  =				""
																continue
													elif "from absl import logging" in out_line:
																lowercase__					  =				"from datasets import logging\n"
													elif "getLogger" in out_line:
																lowercase__					  =				out_line.replace("getLogger",					"get_logger"						)
													elif any(expression in out_line for expression in TO_HIGHLIGHT						):
																lowercase__					  =				True
																lowercase__					  =				list(filter(lambda __lowercase						: e in out_line,					__lowercase						)						)
																out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(__lowercase						) + "\n"						)
																out_lines.append(__lowercase						)
																out_lines.append(__lowercase						)
																continue
													else:
																for pattern, replacement in TO_CONVERT:
																			lowercase__					  =				re.sub(__lowercase,					__lowercase,					__lowercase						)

                # Take care of saving utilities (to later move them together with main script)
													if "tensorflow_datasets" in out_line:
																lowercase__					  =				re.match(R"from\stensorflow_datasets.*import\s([^\.\r\n]+)",					__lowercase						)
																tfds_imports.extend(imp.strip() for imp in match.group(1						).split(","						)						)
																lowercase__					  =				"from . import " + match.group(1						)

													# Check we have not forget anything
													if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
																raise ValueError(F'''Error converting {out_line.strip()}'''						)

													if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
																lowercase__					  =				True
													out_lines.append(__lowercase						)

										if is_builder or "wmt" in f_name:
													# We create a new directory for each dataset
													lowercase__					  =				f_name.replace(".py",					""						)
													lowercase__					  =				os.path.join(__lowercase,					__lowercase						)
													lowercase__					  =				os.path.join(__lowercase,					__lowercase						)
													os.makedirs(__lowercase,					exist_ok=__lowercase						)
													self._logger.info(F'''Adding directory {output_dir}'''						)
													imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}						)
										else:
													# Utilities will be moved at the end
													utils_files.append(__lowercase						)

										if needs_manual_update:
													with_manual_update.append(__lowercase						)

										with open(__lowercase,					"w",					encoding="utf-8"						) as f:
													f.writelines(__lowercase						)
										self._logger.info(F'''Converted in {output_file}'''						)

							for utils_file in utils_files:
										try:
													lowercase__					  =				os.path.basename(__lowercase						)
													lowercase__					  =				imports_to_builder_map[f_name.replace(".py",					""						)]
													self._logger.info(F'''Moving {dest_folder} to {utils_file}'''						)
													shutil.copy(__lowercase,					__lowercase						)
										except KeyError:
													self._logger.error(F'''Cannot find destination folder for {utils_file}. Please copy manually.'''						)

							if with_manual_update:
										for file_path in with_manual_update:
													self._logger.warning(
													    F'''You need to manually update file {file_path} to remove configurations using \'TextEncoderConfig\'.'''						)




