



"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
     from PIL import Image

     from transformers import CLIPImageProcessor, CLIPProcessor



@require_vision
class    _lowerCAmelCase    (				unittest.TestCase		):





  """simple docstring"""






  def    UpperCAmelCase__							(					self   ) ->						List[Any]:







        '''simple docstring'''

        snake_case_   : Dict						=			tempfile.mkdtemp()

        # fmt: off
        snake_case_   : Optional[int]						=			["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
        # fmt: on
        snake_case_   : Tuple						=			dict(zip(_lowercase							,    range(len(_lowercase   )   )   )   )
        snake_case_   : int						=			["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
        snake_case_   : List[str]						=			{"""unk_token""": """<unk>"""}

        snake_case_   : Tuple						=			os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES["""vocab_file"""]   )
        snake_case_   : Optional[Any]						=			os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES["""merges_file"""]   )
        with open(self.vocab_file							,    """w"""							,    encoding="""utf-8"""   ) as fp:
              fp.write(json.dumps(_lowercase   ) + """\n"""   )
        with open(self.merges_file							,    """w"""							,    encoding="""utf-8"""   ) as fp:
              fp.write("""\n""".join(_lowercase   )   )

        snake_case_   : Optional[Any]						=			{
            """do_resize""": True,
            """size""": 2_0,
            """do_center_crop""": True,
            """crop_size""": 1_8,
            """do_normalize""": True,
            """image_mean""": [0.4814_5466, 0.457_8275, 0.4082_1073],
            """image_std""": [0.2686_2954, 0.2613_0258, 0.2757_7711],
        }
        snake_case_   : Tuple						=			os.path.join(self.tmpdirname							,    _lowercase   )
        with open(self.image_processor_file							,    """w"""							,    encoding="""utf-8"""   ) as fp:
              json.dump(_lowercase							,    _lowercase   )





  def    UpperCAmelCase__							(					self							,    **_lowercase   ) ->						str:







        '''simple docstring'''

        return CLIPTokenizer.from_pretrained(self.tmpdirname							,    **_lowercase   )





  def    UpperCAmelCase__							(					self							,    **_lowercase   ) ->						str:







        '''simple docstring'''

        return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,    **_lowercase   )





  def    UpperCAmelCase__							(					self							,    **_lowercase   ) ->						Any:







        '''simple docstring'''

        return CLIPImageProcessor.from_pretrained(self.tmpdirname							,    **_lowercase   )





  def    UpperCAmelCase__							(					self   ) ->						Optional[int]:







        '''simple docstring'''

        shutil.rmtree(self.tmpdirname   )





  def    UpperCAmelCase__							(					self   ) ->						List[str]:







        '''simple docstring'''

        snake_case_   : Tuple						=			[np.random.randint(2_5_5							,    size=(3, 3_0, 4_0_0)							,    dtype=np.uinta   )]

        snake_case_   : Optional[int]						=			[Image.fromarray(np.moveaxis(_lowercase							,    0							,    -1   )   ) for x in image_inputs]

        return image_inputs





  def    UpperCAmelCase__							(					self   ) ->						int:







        '''simple docstring'''

        snake_case_   : Any						=			self.get_tokenizer()
        snake_case_   : Union[str, Any]						=			self.get_rust_tokenizer()
        snake_case_   : List[Any]						=			self.get_image_processor()

        snake_case_   : Dict						=			CLIPProcessor(tokenizer=_lowercase							,    image_processor=_lowercase   )
        processor_slow.save_pretrained(self.tmpdirname   )
        snake_case_   : Optional[Any]						=			CLIPProcessor.from_pretrained(self.tmpdirname							,    use_fast=_lowercase   )

        snake_case_   : Optional[int]						=			CLIPProcessor(tokenizer=_lowercase							,    image_processor=_lowercase   )
        processor_fast.save_pretrained(self.tmpdirname   )
        snake_case_   : int						=			CLIPProcessor.from_pretrained(self.tmpdirname   )

        self.assertEqual(processor_slow.tokenizer.get_vocab()							,    tokenizer_slow.get_vocab()   )
        self.assertEqual(processor_fast.tokenizer.get_vocab()							,    tokenizer_fast.get_vocab()   )
        self.assertEqual(tokenizer_slow.get_vocab()							,    tokenizer_fast.get_vocab()   )
        self.assertIsInstance(processor_slow.tokenizer							,    _lowercase   )
        self.assertIsInstance(processor_fast.tokenizer							,    _lowercase   )

        self.assertEqual(processor_slow.image_processor.to_json_string()							,    image_processor.to_json_string()   )
        self.assertEqual(processor_fast.image_processor.to_json_string()							,    image_processor.to_json_string()   )
        self.assertIsInstance(processor_slow.image_processor							,    _lowercase   )
        self.assertIsInstance(processor_fast.image_processor							,    _lowercase   )





  def    UpperCAmelCase__							(					self   ) ->						List[str]:







        '''simple docstring'''

        snake_case_   : Optional[Any]						=			CLIPProcessor(tokenizer=self.get_tokenizer()							,    image_processor=self.get_image_processor()   )
        processor.save_pretrained(self.tmpdirname   )

        snake_case_   : Tuple						=			self.get_tokenizer(bos_token="""(BOS)"""							,    eos_token="""(EOS)"""   )
        snake_case_   : List[str]						=			self.get_image_processor(do_normalize=_lowercase							,    padding_value=1.0   )

        snake_case_   : int						=			CLIPProcessor.from_pretrained(
            self.tmpdirname							,    bos_token="""(BOS)"""							,    eos_token="""(EOS)"""							,    do_normalize=_lowercase							,    padding_value=1.0   )

        self.assertEqual(processor.tokenizer.get_vocab()							,    tokenizer_add_kwargs.get_vocab()   )
        self.assertIsInstance(processor.tokenizer							,    _lowercase   )

        self.assertEqual(processor.image_processor.to_json_string()							,    image_processor_add_kwargs.to_json_string()   )
        self.assertIsInstance(processor.image_processor							,    _lowercase   )





  def    UpperCAmelCase__							(					self   ) ->						str:







        '''simple docstring'''

        snake_case_   : Optional[Any]						=			self.get_image_processor()
        snake_case_   : Dict						=			self.get_tokenizer()

        snake_case_   : Dict						=			CLIPProcessor(tokenizer=_lowercase							,    image_processor=_lowercase   )

        snake_case_   : str						=			self.prepare_image_inputs()

        snake_case_   : Union[str, Any]						=			image_processor(_lowercase							,    return_tensors="""np"""   )
        snake_case_   : Any						=			processor(images=_lowercase							,    return_tensors="""np"""   )

        for key in input_image_proc.keys():
              self.assertAlmostEqual(input_image_proc[key].sum()							,    input_processor[key].sum()							,    delta=1E-2   )





  def    UpperCAmelCase__							(					self   ) ->						Tuple:







        '''simple docstring'''

        snake_case_   : List[Any]						=			self.get_image_processor()
        snake_case_   : List[str]						=			self.get_tokenizer()

        snake_case_   : Optional[int]						=			CLIPProcessor(tokenizer=_lowercase							,    image_processor=_lowercase   )

        snake_case_   : int						=			"""lower newer"""

        snake_case_   : List[str]						=			processor(text=_lowercase   )

        snake_case_   : Optional[int]						=			tokenizer(_lowercase   )

        for key in encoded_tok.keys():
              self.assertListEqual(encoded_tok[key]							,    encoded_processor[key]   )





  def    UpperCAmelCase__							(					self   ) ->						str:







        '''simple docstring'''

        snake_case_   : int						=			self.get_image_processor()
        snake_case_   : str						=			self.get_tokenizer()

        snake_case_   : Union[str, Any]						=			CLIPProcessor(tokenizer=_lowercase							,    image_processor=_lowercase   )

        snake_case_   : str						=			"""lower newer"""
        snake_case_   : Tuple						=			self.prepare_image_inputs()

        snake_case_   : Dict						=			processor(text=_lowercase							,    images=_lowercase   )

        self.assertListEqual(list(inputs.keys()   )							,    ["""input_ids""", """attention_mask""", """pixel_values"""]   )

        # test if it raises when no input is passed
        with pytest.raises(_lowercase   ):
              processor()





  def    UpperCAmelCase__							(					self   ) ->						List[str]:







        '''simple docstring'''

        snake_case_   : Tuple						=			self.get_image_processor()
        snake_case_   : Optional[Any]						=			self.get_tokenizer()

        snake_case_   : Optional[int]						=			CLIPProcessor(tokenizer=_lowercase							,    image_processor=_lowercase   )

        snake_case_   : Tuple						=			[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        snake_case_   : str						=			processor.batch_decode(_lowercase   )
        snake_case_   : Any						=			tokenizer.batch_decode(_lowercase   )

        self.assertListEqual(_lowercase							,    _lowercase   )





  def    UpperCAmelCase__							(					self   ) ->						str:







        '''simple docstring'''

        snake_case_   : Dict						=			self.get_image_processor()
        snake_case_   : int						=			self.get_tokenizer()

        snake_case_   : List[Any]						=			CLIPProcessor(tokenizer=_lowercase							,    image_processor=_lowercase   )

        snake_case_   : Union[str, Any]						=			"""lower newer"""
        snake_case_   : Optional[Any]						=			self.prepare_image_inputs()

        snake_case_   : Optional[Any]						=			processor(text=_lowercase							,    images=_lowercase   )

        self.assertListEqual(list(inputs.keys()   )							,    processor.model_input_names   )






"""simple docstring"""





from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)

