






'''simple docstring'''



from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers.models.xglm.modeling_tf_xglm import (
          TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFXGLMForCausalLM,
          TFXGLMModel,
      )
@require_tf
class 					UpperCAmelCase_					:




      """simple docstring"""




      UpperCamelCase_								=			XGLMConfig
      UpperCamelCase_								=			{}
      UpperCamelCase_								=			'gelu'






      def __init__( self			:		Optional[int]     ,      UpperCAmelCase			:		Optional[int]     ,      UpperCAmelCase			:		Union[str, Any]=14     ,      UpperCAmelCase			:		Any=7     ,      UpperCAmelCase			:		List[Any]=True     ,      UpperCAmelCase			:		List[str]=True     ,      UpperCAmelCase			:		Any=True     ,      UpperCAmelCase			:		str=99     ,      UpperCAmelCase			:		Dict=32     ,      UpperCAmelCase			:		int=2     ,      UpperCAmelCase			:		Tuple=4     ,      UpperCAmelCase			:		Optional[Any]=37     ,      UpperCAmelCase			:		str="gelu"     ,      UpperCAmelCase			:		Tuple=0.1     ,      UpperCAmelCase			:		Union[str, Any]=0.1     ,      UpperCAmelCase			:		Optional[int]=512     ,      UpperCAmelCase			:		Any=0.0_2     ,      )  ->  str:


       '''simple docstring'''

       lowercase					:   Any				  =parent
       lowercase					:   Tuple				  =batch_size
       lowercase					:   Dict				  =seq_length
       lowercase					:   List[str]				  =is_training
       lowercase					:   int				  =use_input_mask
       lowercase					:   Any				  =use_labels
       lowercase					:   str				  =vocab_size
       lowercase					:   Dict				  =d_model
       lowercase					:   str				  =num_hidden_layers
       lowercase					:   Optional[int]				  =num_attention_heads
       lowercase					:   List[Any]				  =ffn_dim
       lowercase					:   Optional[int]				  =activation_function
       lowercase					:   int				  =activation_dropout
       lowercase					:   Optional[Any]				  =attention_dropout
       lowercase					:   List[str]				  =max_position_embeddings
       lowercase					:   Optional[int]				  =initializer_range
       lowercase					:   str				  =None
       lowercase					:   Any				  =0
       lowercase					:   Union[str, Any]				  =2
       lowercase					:   Optional[Any]				  =1






      def 				A__      ( self			:		Any		)  ->  int:


       '''simple docstring'''

       return XGLMConfig.from_pretrained('''facebook/xglm-564M'''		)






      def 				A__      ( self			:		Any		)  ->  Any:


       '''simple docstring'''

       lowercase					:   List[str]				  =tf.clip_by_value(
           ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size		)     ,      clip_value_min=0     ,      clip_value_max=3		)

       lowercase					:   Optional[Any]				  =None
       if self.use_input_mask:
        lowercase					:   int				  =random_attention_mask([self.batch_size, self.seq_length]		)

       lowercase					:   str				  =self.get_config()

       lowercase					:   List[Any]				  =floats_tensor([self.num_hidden_layers, self.num_attention_heads]     ,      2		)

       return (
           config,
           input_ids,
           input_mask,
           head_mask,
       )






      def 				A__      ( self			:		Dict		)  ->  Any:


       '''simple docstring'''

       return XGLMConfig(
           vocab_size=self.vocab_size     ,      d_model=self.hidden_size     ,      num_layers=self.num_hidden_layers     ,      attention_heads=self.num_attention_heads     ,      ffn_dim=self.ffn_dim     ,      activation_function=self.activation_function     ,      activation_dropout=self.activation_dropout     ,      attention_dropout=self.attention_dropout     ,      max_position_embeddings=self.max_position_embeddings     ,      initializer_range=self.initializer_range     ,      use_cache=_lowercase     ,      bos_token_id=self.bos_token_id     ,      eos_token_id=self.eos_token_id     ,      pad_token_id=self.pad_token_id     ,      return_dict=_lowercase     ,      )






      def 				A__      ( self			:		Tuple		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   str				  =self.prepare_config_and_inputs()

       (
           lowercase      
       )					:   Dict				  =config_and_inputs

       lowercase					:   Optional[Any]				  ={
           'input_ids': input_ids,
           'head_mask': head_mask,
       }

       return config, inputs_dict





@require_tf
class 					UpperCAmelCase_					(    UpperCAmelCase_				,       UpperCAmelCase_				,       unittest.TestCase				):




      """simple docstring"""




      UpperCamelCase_								=			(TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
      UpperCamelCase_								=			(TFXGLMForCausalLM,) if is_tf_available() else ()
      UpperCamelCase_								=			(
          {'feature-extraction': TFXGLMModel, 'text-generation': TFXGLMForCausalLM} if is_tf_available() else {}
      )
      UpperCamelCase_								=			False
      UpperCamelCase_								=			False
      UpperCamelCase_								=			False






      def 				A__      ( self			:		Union[str, Any]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =TFXGLMModelTester(self		)
       lowercase					:   List[str]				  =ConfigTester(self     ,      config_class=_lowercase     ,      n_embd=37		)






      def 				A__      ( self			:		Any		)  ->  List[Any]:


       '''simple docstring'''

       self.config_tester.run_common_tests()






      @slow
      def 				A__      ( self			:		Optional[Any]		)  ->  int:


       '''simple docstring'''

       for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowercase					:   Any				  =TFXGLMModel.from_pretrained(_lowercase		)
        self.assertIsNotNone(_lowercase		)






      @unittest.skip(reason='''Currently, model embeddings are going to undergo a major refactor.'''		)
      def 				A__      ( self			:		List[str]		)  ->  List[Any]:


       '''simple docstring'''

       super().test_resize_token_embeddings()





@require_tf
class 					UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      @slow
      def 				A__      ( self			:		str     ,      UpperCAmelCase			:		Union[str, Any]=True		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   List[str]				  =TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''		)
       lowercase					:   Tuple				  =tf.convert_to_tensor([[2, 268, 9865]]     ,      dtype=tf.intaa		)  # The dog
       # </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
       # fmt: off
       lowercase					:   Any				  =[2, 268, 9865, 67, 11, 1988, 5_7252, 9865, 5, 984, 67, 1988, 21_3838, 1658, 53, 7_0446, 33, 6657, 278, 1581]
       # fmt: on
       lowercase					:   Tuple				  =model.generate(_lowercase     ,      do_sample=_lowercase     ,      num_beams=1		)
       if verify_outputs:
        self.assertListEqual(output_ids[0].numpy().tolist()     ,      _lowercase		)






      @slow
      def 				A__      ( self			:		Optional[int]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   int				  =XGLMTokenizer.from_pretrained('''facebook/xglm-564M'''		)
       lowercase					:   Dict				  =TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''		)

       tf.random.set_seed(0		)
       lowercase					:   Any				  =tokenizer('''Today is a nice day and'''     ,      return_tensors='''tf'''		)
       lowercase					:   Any				  =tokenized.input_ids
       # forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
       with tf.device(''':/CPU:0'''		):
        lowercase					:   List[str]				  =model.generate(_lowercase     ,      do_sample=_lowercase     ,      seed=[7, 0]		)
       lowercase					:   Optional[Any]				  =tokenizer.decode(output_ids[0]     ,      skip_special_tokens=_lowercase		)

       lowercase					:   Optional[Any]				  =(
           'Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due'
       )
       self.assertEqual(_lowercase     ,      _lowercase		)






      @slow
      def 				A__      ( self			:		Any		)  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''		)
       lowercase					:   Optional[Any]				  =XGLMTokenizer.from_pretrained('''facebook/xglm-564M'''		)

       lowercase					:   str				  ='left'

       # use different length sentences to test batching
       lowercase					:   str				  =[
           'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
           'left-padding, such as in batched generation. The output for the sequence below should be the same '
           'regardless of whether left padding is applied or not. When',
           'Hello, my dog is a little',
       ]

       lowercase					:   List[Any]				  =tokenizer(_lowercase     ,      return_tensors='''tf'''     ,      padding=_lowercase		)
       lowercase					:   Tuple				  =inputs['input_ids']

       lowercase					:   Optional[int]				  =model.generate(input_ids=_lowercase     ,      attention_mask=inputs['''attention_mask''']     ,      max_new_tokens=12		)

       lowercase					:   Optional[Any]				  =tokenizer(sentences[0]     ,      return_tensors='''tf'''		).input_ids
       lowercase					:   List[Any]				  =model.generate(input_ids=_lowercase     ,      max_new_tokens=12		)

       lowercase					:   List[Any]				  =tokenizer(sentences[1]     ,      return_tensors='''tf'''		).input_ids
       lowercase					:   Tuple				  =model.generate(input_ids=_lowercase     ,      max_new_tokens=12		)

       lowercase					:   Union[str, Any]				  =tokenizer.batch_decode(_lowercase     ,      skip_special_tokens=_lowercase		)
       lowercase					:   Optional[Any]				  =tokenizer.decode(output_non_padded[0]     ,      skip_special_tokens=_lowercase		)
       lowercase					:   Tuple				  =tokenizer.decode(output_padded[0]     ,      skip_special_tokens=_lowercase		)

       lowercase					:   Dict				  =[
           'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
           'left-padding, such as in batched generation. The output for the sequence below should be the same '
           'regardless of whether left padding is applied or not. When left padding is applied, the sequence will be '
           'a single',
           'Hello, my dog is a little bit of a shy one, but he is very friendly',
       ]
       self.assertListEqual(_lowercase     ,      _lowercase		)
       self.assertListEqual(_lowercase     ,      [non_padded_sentence, padded_sentence]		)











'''simple docstring'''



import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			['''image_processor''', '''tokenizer''']
      UpperCamelCase_								=			'''LayoutLMv2ImageProcessor'''
      UpperCamelCase_								=			('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')






      def __init__( self			:		List[str]     ,      UpperCAmelCase			:		Tuple=None     ,      UpperCAmelCase			:		str=None     ,      **UpperCAmelCase			:		Dict		)  ->  Optional[int]:


       '''simple docstring'''

       if "feature_extractor" in kwargs:
        warnings.warn(
            '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
            ''' instead.'''     ,      UpperCAmelCase     ,      )
        lowercase					:   Any				  =kwargs.pop('''feature_extractor'''		)

       lowercase					:   Dict				  =image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
        raise ValueError('''You need to specify an `image_processor`.'''		)
       if tokenizer is None:
        raise ValueError('''You need to specify a `tokenizer`.'''		)

       super().__init__(UpperCAmelCase     ,      UpperCAmelCase		)






      def __call__( self			:		Any     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None     ,      UpperCAmelCase			:		Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None     ,      UpperCAmelCase			:		Union[List[List[int]], List[List[List[int]]]] = None     ,      UpperCAmelCase			:		Optional[Union[List[int], List[List[int]]]] = None     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		Union[bool, str, PaddingStrategy] = False     ,      UpperCAmelCase			:		Union[bool, str, TruncationStrategy] = None     ,      UpperCAmelCase			:		Optional[int] = None     ,      UpperCAmelCase			:		int = 0     ,      UpperCAmelCase			:		Optional[int] = None     ,      UpperCAmelCase			:		Optional[bool] = None     ,      UpperCAmelCase			:		Optional[bool] = None     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		Optional[Union[str, TensorType]] = None     ,      **UpperCAmelCase			:		Any     ,      )  ->  BatchEncoding:


       '''simple docstring'''

       if self.image_processor.apply_ocr and (boxes is not None):
        raise ValueError(
            '''You cannot provide bounding boxes '''
            '''if you initialized the image processor with apply_ocr set to True.'''		)

       if self.image_processor.apply_ocr and (word_labels is not None):
        raise ValueError(
            '''You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'''		)

       if return_overflowing_tokens is True and return_offsets_mapping is False:
        raise ValueError('''You cannot return overflowing tokens without returning the offsets mapping.'''		)

       # first, apply the image processor
       lowercase					:   Tuple				  =self.image_processor(images=UpperCAmelCase     ,      return_tensors=UpperCAmelCase		)

       # second, apply the tokenizer
       if text is not None and self.image_processor.apply_ocr and text_pair is None:
        if isinstance(UpperCAmelCase     ,      UpperCAmelCase		):
         lowercase					:   Optional[Any]				  =[text]  # add batch dimension (as the image processor always adds a batch dimension)
        lowercase					:   List[str]				  =features['''words''']

       lowercase					:   Optional[Any]				  =self.tokenizer(
           text=text if text is not None else features['''words''']     ,      text_pair=text_pair if text_pair is not None else None     ,      boxes=boxes if boxes is not None else features['''boxes''']     ,      word_labels=UpperCAmelCase     ,      add_special_tokens=UpperCAmelCase     ,      padding=UpperCAmelCase     ,      truncation=UpperCAmelCase     ,      max_length=UpperCAmelCase     ,      stride=UpperCAmelCase     ,      pad_to_multiple_of=UpperCAmelCase     ,      return_token_type_ids=UpperCAmelCase     ,      return_attention_mask=UpperCAmelCase     ,      return_overflowing_tokens=UpperCAmelCase     ,      return_special_tokens_mask=UpperCAmelCase     ,      return_offsets_mapping=UpperCAmelCase     ,      return_length=UpperCAmelCase     ,      verbose=UpperCAmelCase     ,      return_tensors=UpperCAmelCase     ,      **UpperCAmelCase     ,      )

       # add pixel values
       lowercase					:   List[str]				  =features.pop('''pixel_values'''		)
       if return_overflowing_tokens is True:
        lowercase					:   str				  =self.get_overflowing_images(UpperCAmelCase     ,      encoded_inputs['''overflow_to_sample_mapping''']		)
       lowercase					:   Dict				  =images

       return encoded_inputs






      def 				A__      ( self			:		Any     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		str		)  ->  str:


       '''simple docstring'''

       lowercase					:   str				  =[]
       for sample_idx in overflow_to_sample_mapping:
        images_with_overflow.append(images[sample_idx]		)

       if len(UpperCAmelCase		) != len(UpperCAmelCase		):
        raise ValueError(
            '''Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'''
            f' {len(UpperCAmelCase		)} and {len(UpperCAmelCase		)}'		)

       return images_with_overflow






      def 				A__      ( self			:		Union[str, Any]     ,      *UpperCAmelCase			:		Tuple     ,      **UpperCAmelCase			:		Dict		)  ->  int:


       '''simple docstring'''

       return self.tokenizer.batch_decode(*UpperCAmelCase     ,      **UpperCAmelCase		)






      def 				A__      ( self			:		Dict     ,      *UpperCAmelCase			:		Any     ,      **UpperCAmelCase			:		Dict		)  ->  str:


       '''simple docstring'''

       return self.tokenizer.decode(*UpperCAmelCase     ,      **UpperCAmelCase		)






      @property
      def 				A__      ( self			:		List[str]		)  ->  List[str]:


       '''simple docstring'''

       return ["input_ids", "bbox", "attention_mask", "image"]






      @property
      def 				A__      ( self			:		int		)  ->  List[str]:


       '''simple docstring'''

       warnings.warn(
           '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''     ,      UpperCAmelCase     ,      )
       return self.image_processor_class






      @property
      def 				A__      ( self			:		Dict		)  ->  List[str]:


       '''simple docstring'''

       warnings.warn(
           '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''     ,      UpperCAmelCase     ,      )
       return self.image_processor



