



from __future__ import annotations

import unittest

import numpy as np

from transformers import LayoutLMConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers.models.layoutlm.modeling_tf_layoutlm import (
							    TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
							    TFLayoutLMForMaskedLM,
							    TFLayoutLMForQuestionAnswering,
							    TFLayoutLMForSequenceClassification,
							    TFLayoutLMForTokenClassification,
							    TFLayoutLMModel,
							)
class 				_snake_case			:




				def __init__(						self					:						Optional[int],					__lowercase					:						Any,					__lowercase					:						Optional[int]=13,					__lowercase					:						Tuple=7,					__lowercase					:						List[Any]=True,					__lowercase					:						int=True,					__lowercase					:						Dict=True,					__lowercase					:						List[Any]=True,					__lowercase					:						Optional[int]=99,					__lowercase					:						Tuple=32,					__lowercase					:						Dict=2,					__lowercase					:						Any=4,					__lowercase					:						Dict=37,					__lowercase					:						List[Any]="gelu",					__lowercase					:						str=0.1,					__lowercase					:						Optional[int]=0.1,					__lowercase					:						int=512,					__lowercase					:						Any=16,					__lowercase					:						List[str]=2,					__lowercase					:						Union[str, Any]=0.02,					__lowercase					:						str=3,					__lowercase					:						str=4,					__lowercase					:						Tuple=None,					__lowercase					:						List[str]=1000,					):
							lowercase__					  =				parent
							lowercase__					  =				batch_size
							lowercase__					  =				seq_length
							lowercase__					  =				is_training
							lowercase__					  =				use_input_mask
							lowercase__					  =				use_token_type_ids
							lowercase__					  =				use_labels
							lowercase__					  =				vocab_size
							lowercase__					  =				hidden_size
							lowercase__					  =				num_hidden_layers
							lowercase__					  =				num_attention_heads
							lowercase__					  =				intermediate_size
							lowercase__					  =				hidden_act
							lowercase__					  =				hidden_dropout_prob
							lowercase__					  =				attention_probs_dropout_prob
							lowercase__					  =				max_position_embeddings
							lowercase__					  =				type_vocab_size
							lowercase__					  =				type_sequence_label_size
							lowercase__					  =				initializer_range
							lowercase__					  =				num_labels
							lowercase__					  =				num_choices
							lowercase__					  =				scope
							lowercase__					  =				range_bbox




				def 							A__      (						self					:						Dict						):
							lowercase__					  =				ids_tensor([self.batch_size, self.seq_length],					self.vocab_size						)

							# convert bbox to numpy since TF does not support item assignment
							lowercase__					  =				ids_tensor([self.batch_size, self.seq_length, 4],					self.range_bbox						).numpy()
							# Ensure that bbox is legal
							for i in range(bbox.shape[0]						):
										for j in range(bbox.shape[1]						):
													if bbox[i, j, 3] < bbox[i, j, 1]:
																lowercase__					  =				bbox[i, j, 3]
																lowercase__					  =				bbox[i, j, 1]
																lowercase__					  =				t
													if bbox[i, j, 2] < bbox[i, j, 0]:
																lowercase__					  =				bbox[i, j, 2]
																lowercase__					  =				bbox[i, j, 0]
																lowercase__					  =				t
							lowercase__					  =				tf.convert_to_tensor(__lowercase						)

							lowercase__					  =				None
							if self.use_input_mask:
										lowercase__					  =				random_attention_mask([self.batch_size, self.seq_length]						)

							lowercase__					  =				None
							if self.use_token_type_ids:
										lowercase__					  =				ids_tensor([self.batch_size, self.seq_length],					self.type_vocab_size						)

							lowercase__					  =				None
							lowercase__					  =				None
							lowercase__					  =				None
							if self.use_labels:
										lowercase__					  =				ids_tensor([self.batch_size],					self.type_sequence_label_size						)
										lowercase__					  =				ids_tensor([self.batch_size, self.seq_length],					self.num_labels						)
										lowercase__					  =				ids_tensor([self.batch_size],					self.num_choices						)

							lowercase__					  =				LayoutLMConfig(
							    vocab_size=self.vocab_size,					hidden_size=self.hidden_size,					num_hidden_layers=self.num_hidden_layers,					num_attention_heads=self.num_attention_heads,					intermediate_size=self.intermediate_size,					hidden_act=self.hidden_act,					hidden_dropout_prob=self.hidden_dropout_prob,					attention_probs_dropout_prob=self.attention_probs_dropout_prob,					max_position_embeddings=self.max_position_embeddings,					type_vocab_size=self.type_vocab_size,					initializer_range=self.initializer_range,					)

							return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def 							A__      (						self					:						List[Any],					__lowercase					:						List[Any],					__lowercase					:						Any,					__lowercase					:						Optional[int],					__lowercase					:						Optional[int],					__lowercase					:						Union[str, Any],					__lowercase					:						Optional[Any],					__lowercase					:						List[Any],					__lowercase					:						List[str]						):
							lowercase__					  =				TFLayoutLMModel(config=__lowercase						)

							lowercase__					  =				model(__lowercase,					__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase						)
							lowercase__					  =				model(__lowercase,					__lowercase,					token_type_ids=__lowercase						)
							lowercase__					  =				model(__lowercase,					__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size)						)
							self.parent.assertEqual(result.pooler_output.shape,					(self.batch_size, self.hidden_size)						)




				def 							A__      (						self					:						List[Any],					__lowercase					:						Union[str, Any],					__lowercase					:						int,					__lowercase					:						Union[str, Any],					__lowercase					:						Tuple,					__lowercase					:						Optional[Any],					__lowercase					:						List[Any],					__lowercase					:						str,					__lowercase					:						List[Any]						):
							lowercase__					  =				TFLayoutLMForMaskedLM(config=__lowercase						)

							lowercase__					  =				model(__lowercase,					__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase,					labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.vocab_size)						)




				def 							A__      (						self					:						str,					__lowercase					:						int,					__lowercase					:						int,					__lowercase					:						Optional[Any],					__lowercase					:						int,					__lowercase					:						List[str],					__lowercase					:						List[Any],					__lowercase					:						int,					__lowercase					:						str						):
							lowercase__					  =				self.num_labels
							lowercase__					  =				TFLayoutLMForSequenceClassification(config=__lowercase						)

							lowercase__					  =				model(__lowercase,					__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase						)
							self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_labels)						)




				def 							A__      (						self					:						Any,					__lowercase					:						List[str],					__lowercase					:						List[Any],					__lowercase					:						str,					__lowercase					:						int,					__lowercase					:						Tuple,					__lowercase					:						Tuple,					__lowercase					:						Any,					__lowercase					:						Dict						):
							lowercase__					  =				self.num_labels
							lowercase__					  =				TFLayoutLMForTokenClassification(config=__lowercase						)

							lowercase__					  =				model(__lowercase,					__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase,					labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.num_labels)						)




				def 							A__      (						self					:						Union[str, Any],					__lowercase					:						int,					__lowercase					:						List[str],					__lowercase					:						int,					__lowercase					:						Optional[int],					__lowercase					:						Dict,					__lowercase					:						Tuple,					__lowercase					:						Any,					__lowercase					:						int						):
							lowercase__					  =				TFLayoutLMForQuestionAnswering(config=__lowercase						)

							lowercase__					  =				model(__lowercase,					__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase						)
							self.parent.assertEqual(result.start_logits.shape,					(self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape,					(self.batch_size, self.seq_length)						)




				def 							A__      (						self					:						int						):
							lowercase__					  =				self.prepare_config_and_inputs()
							(
							    (
							    lowercase__       
							)  ,       (
							    lowercase__       
							)  ,       (
							    lowercase__       
							)  ,       (
							    lowercase__       
							)  ,       (
							    lowercase__       
							)  ,       (
							    lowercase__       
							)  ,       (
							    lowercase__       
							)  ,       (
							    lowercase__       
							)  ,              
							)					  =				config_and_inputs
							lowercase__					  =				{
							    "input_ids": input_ids,
							    "bbox": bbox,
							    "token_type_ids": token_type_ids,
							    "attention_mask": input_mask,
							}
							return config, inputs_dict
@require_tf
class 				_snake_case			(					lowercase__    ,	lowercase__    ,	unittest.TestCase):
				UpperCamelCase__				:					Optional[Any]     		=(
				    (
				        TFLayoutLMModel,
				        TFLayoutLMForMaskedLM,
				        TFLayoutLMForTokenClassification,
				        TFLayoutLMForSequenceClassification,
				        TFLayoutLMForQuestionAnswering,
				    )
				    if is_tf_available()
				    else ()
				)
				UpperCamelCase__				:					Tuple     		=(
				    {
				        """feature-extraction""": TFLayoutLMModel,
				        """fill-mask""": TFLayoutLMForMaskedLM,
				        """text-classification""": TFLayoutLMForSequenceClassification,
				        """token-classification""": TFLayoutLMForTokenClassification,
				        """zero-shot""": TFLayoutLMForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				UpperCamelCase__				:					int     		=False
				UpperCamelCase__				:					Union[str, Any]     		=True
				UpperCamelCase__				:					str     		=1_0




				def 							A__      (						self					:						Optional[Any]						):
							lowercase__					  =				TFLayoutLMModelTester(self						)
							lowercase__					  =				ConfigTester(self,					config_class=__lowercase,					hidden_size=37						)




				def 							A__      (						self					:						List[str]						):
							self.config_tester.run_common_tests()




				def 							A__      (						self					:						List[str]						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowercase						)




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__lowercase						)




				def 							A__      (						self					:						Dict						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__lowercase						)




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__lowercase						)




				def 							A__      (						self					:						Optional[int]						):
							lowercase__					  =				self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__lowercase						)




				@slow
				def 							A__      (						self					:						Optional[Any]						):
							for model_name in TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										lowercase__					  =				TFLayoutLMModel.from_pretrained(__lowercase						)
										self.assertIsNotNone(__lowercase						)




				@unittest.skip("Onnx compliancy broke with TF 2.10"						)
				def 							A__      (						self					:						Dict						):
							pass



def __lowerCAmelCase (     ):
			# Here we prepare a batch of 2 sequences to test a LayoutLM forward pass on:
			# fmt: off
			lowercase__					  =				tf.convert_to_tensor([[101,1019,1014,1016,1037,1_2849,4747,1004,1_4246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,1_1300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,1_9274,2772,6205,2_7814,1_6147,1_6147,4343,2047,1_0283,1_0969,1_4389,1012,2338,102]]  )  # noqa: E231
			lowercase__					  =				tf.convert_to_tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]  )  # noqa: E231
			lowercase__					  =				tf.convert_to_tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]]  )  # noqa: E231
			lowercase__					  =				tf.convert_to_tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]  )  # noqa: E231
			# these are sequence labels (i.e. at the token level)
			lowercase__					  =				tf.convert_to_tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]  )  # noqa: E231
			# fmt: on

			return input_ids, attention_mask, bbox, token_type_ids, labels




@require_tf
class 				_snake_case			(					unittest.TestCase):




				@slow
				def 							A__      (						self					:						str						):
							lowercase__					  =				TFLayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased"						)

							lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__					  =				prepare_layoutlm_batch_inputs()

							# forward pass
							lowercase__					  =				model(input_ids=__lowercase,					bbox=__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase						)

							# test the sequence output on [0, :3, :3]
							lowercase__					  =				tf.convert_to_tensor(
							    [[0.1785, -0.1947, -0.0425], [-0.3254, -0.2807, 0.2553], [-0.5391, -0.3322, 0.3364]],					)

							self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3],					__lowercase,					atol=1e-3						)						)

							# test the pooled output on [1, :3]
							lowercase__					  =				tf.convert_to_tensor([-0.6580, -0.0214, 0.8552]						)

							self.assertTrue(np.allclose(outputs.pooler_output[1, :3],					__lowercase,					atol=1e-3						)						)




				@slow
				def 							A__      (						self					:						Optional[Any]						):
							# initialize model with randomly initialized sequence classification head
							lowercase__					  =				TFLayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased",					num_labels=2						)

							lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__					  =				prepare_layoutlm_batch_inputs()

							# forward pass
							lowercase__					  =				model(
							    input_ids=__lowercase,					bbox=__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase,					labels=tf.convert_to_tensor([1, 1]						),					)

							# test whether we get a loss as a scalar
							lowercase__					  =				outputs.loss
							lowercase__					  =				(2,)
							self.assertEqual(loss.shape,					__lowercase						)

							# test the shape of the logits
							lowercase__					  =				outputs.logits
							lowercase__					  =				(2, 2)
							self.assertEqual(logits.shape,					__lowercase						)




				@slow
				def 							A__      (						self					:						List[str]						):
							# initialize model with randomly initialized token classification head
							lowercase__					  =				TFLayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased",					num_labels=13						)

							lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__					  =				prepare_layoutlm_batch_inputs()

							# forward pass
							lowercase__					  =				model(
							    input_ids=__lowercase,					bbox=__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase,					labels=__lowercase						)

							# test the shape of the logits
							lowercase__					  =				outputs.logits
							lowercase__					  =				tf.convert_to_tensor((2, 25, 13)						)
							self.assertEqual(logits.shape,					__lowercase						)




				@slow
				def 							A__      (						self					:						Optional[Any]						):
							# initialize model with randomly initialized token classification head
							lowercase__					  =				TFLayoutLMForQuestionAnswering.from_pretrained("microsoft/layoutlm-base-uncased"						)

							lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__					  =				prepare_layoutlm_batch_inputs()

							# forward pass
							lowercase__					  =				model(input_ids=__lowercase,					bbox=__lowercase,					attention_mask=__lowercase,					token_type_ids=__lowercase						)

							# test the shape of the logits
							lowercase__					  =				tf.convert_to_tensor((2, 25)						)
							self.assertEqual(outputs.start_logits.shape,					__lowercase						)
							self.assertEqual(outputs.end_logits.shape,					__lowercase						)









import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing
class 				_snake_case			(					lowercase__):




				def __init__(						self					:						Optional[Any],					__lowercase					:						str = "▁",					__lowercase					:						bool = True,					__lowercase					:						Union[str, AddedToken] = "<unk>",					__lowercase					:						Union[str, AddedToken] = "</s>",					__lowercase					:						Union[str, AddedToken] = "<pad>",					):
							lowercase__					  =				{
							    "pad": {"id": 0, "token": pad_token},
							    "eos": {"id": 1, "token": eos_token},
							    "unk": {"id": 2, "token": unk_token},
							}

							lowercase__					  =				[None] * len(self.special_tokens						)
							for token_dict in self.special_tokens.values():
										lowercase__					  =				token_dict["token"]

							lowercase__					  =				Tokenizer(Unigram()						)

							lowercase__					  =				normalizers.Sequence(
							    [
							        normalizers.Nmt(),
							        normalizers.NFKC(),
							        normalizers.Replace(Regex(" {2,}"						),					" "						),
							        normalizers.Lowercase(),
							    ]						)
							lowercase__					  =				pre_tokenizers.Sequence(
							    [
							        pre_tokenizers.Metaspace(replacement=__lowercase,					add_prefix_space=__lowercase						),
							        pre_tokenizers.Digits(individual_digits=__lowercase						),
							        pre_tokenizers.Punctuation(),
							    ]						)
							lowercase__					  =				decoders.Metaspace(replacement=__lowercase,					add_prefix_space=__lowercase						)

							lowercase__					  =				TemplateProcessing(
							    single=F'''$A {self.special_tokens["eos"]["token"]}''',					special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])],					)

							lowercase__					  =				{
							    "model": "SentencePieceUnigram",
							    "replacement": replacement,
							    "add_prefix_space": add_prefix_space,
							}

							super().__init__(__lowercase,					__lowercase						)




				def 							A__      (						self					:						Union[str, Any],					__lowercase					:						Union[str, List[str]],					__lowercase					:						int = 8000,					__lowercase					:						bool = True,					):
							lowercase__					  =				trainers.UnigramTrainer(
							    vocab_size=__lowercase,					special_tokens=self.special_tokens_list,					show_progress=__lowercase,					)

							if isinstance(__lowercase,					__lowercase						):
										lowercase__					  =				[files]
							self._tokenizer.train(__lowercase,					trainer=__lowercase						)

							self.add_unk_id()




				def 							A__      (						self					:						List[Any],					__lowercase					:						Union[Iterator[str], Iterator[Iterator[str]]],					__lowercase					:						int = 8000,					__lowercase					:						bool = True,					):
							lowercase__					  =				trainers.UnigramTrainer(
							    vocab_size=__lowercase,					special_tokens=self.special_tokens_list,					show_progress=__lowercase,					)

							self._tokenizer.train_from_iterator(__lowercase,					trainer=__lowercase						)

							self.add_unk_id()




				def 							A__      (						self					:						str						):
							lowercase__					  =				json.loads(self._tokenizer.to_str()						)

							lowercase__					  =				self.special_tokens["unk"]["id"]

							lowercase__					  =				Tokenizer.from_str(json.dumps(__lowercase						)						)




