






'''simple docstring'''







import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam




class 		UpperCamelCase__    (datasets.BeamBasedBuilder			):




  '''simple docstring'''





  def 			UpperCamelCase_					(      self       ):
        return datasets.DatasetInfo(
            features=datasets.Features({"""content""": datasets.Value("""string"""       )}       )   ,supervised_keys=_lowerCAmelCase   ,)






  def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,gen_kwargs={"""examples""": get_test_dummy_examples()}       )]






  def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
        import apache_beam as beam

        return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase       )




class 		UpperCamelCase__    (datasets.BeamBasedBuilder			):




  '''simple docstring'''





  def 			UpperCamelCase_					(      self       ):
        return datasets.DatasetInfo(
            features=datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""       )}       )}       )   ,supervised_keys=_lowerCAmelCase   ,)






  def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
        return [
            datasets.SplitGenerator(name=datasets.Split.TRAIN   ,gen_kwargs={"""examples""": get_test_nested_examples()}       )
        ]






  def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
        import apache_beam as beam

        return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase       )



def 						A__			(				):
      return [(i, {"content": content}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]						)]
def 						A__			(				):
      return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]						)]





class 		UpperCamelCase__    (a			):




  '''simple docstring'''





  @require_beam
  def 			UpperCamelCase_					(      self       ):
        lowerCamelCase__         =  len(get_test_dummy_examples()       )
        with tempfile.TemporaryDirectory() as tmp_cache_dir:
              lowerCamelCase__         =  DummyBeamDataset(cache_dir=_lowerCAmelCase   ,beam_runner="""DirectRunner"""       )
              builder.download_and_prepare()
              self.assertTrue(
                  os.path.exists(
                      os.path.join(_lowerCAmelCase   ,builder.name   ,"""default"""   ,"""0.0.0"""   ,F'''{builder.name}-train.arrow'''       )       )       )
              self.assertDictEqual(builder.info.features   ,datasets.Features({"""content""": datasets.Value("""string"""       )}       )       )
              lowerCamelCase__         =  builder.as_dataset()
              self.assertEqual(dset["""train"""].num_rows   ,_lowerCAmelCase       )
              self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples   ,_lowerCAmelCase       )
              self.assertDictEqual(dset["""train"""][0]   ,get_test_dummy_examples()[0][1]       )
              self.assertDictEqual(
                  dset["""train"""][expected_num_examples - 1]   ,get_test_dummy_examples()[expected_num_examples - 1][1]       )
              self.assertTrue(
                  os.path.exists(os.path.join(_lowerCAmelCase   ,builder.name   ,"""default"""   ,"""0.0.0"""   ,"""dataset_info.json"""       )       )       )
              del dset






  @require_beam
  def 			UpperCamelCase_					(      self       ):
        import apache_beam as beam

        lowerCamelCase__         =  beam.io.parquetio.WriteToParquet

        lowerCamelCase__         =  len(get_test_dummy_examples()       )
        with tempfile.TemporaryDirectory() as tmp_cache_dir:
              lowerCamelCase__         =  DummyBeamDataset(cache_dir=_lowerCAmelCase   ,beam_runner="""DirectRunner"""       )
              with patch("""apache_beam.io.parquetio.WriteToParquet"""       ) as write_parquet_mock:
                    lowerCamelCase__         =  partial(_lowerCAmelCase   ,num_shards=2       )
                    builder.download_and_prepare()
              self.assertTrue(
                  os.path.exists(
                      os.path.join(
                          _lowerCAmelCase   ,builder.name   ,"""default"""   ,"""0.0.0"""   ,F'''{builder.name}-train-00000-of-00002.arrow'''       )       )       )
              self.assertTrue(
                  os.path.exists(
                      os.path.join(
                          _lowerCAmelCase   ,builder.name   ,"""default"""   ,"""0.0.0"""   ,F'''{builder.name}-train-00000-of-00002.arrow'''       )       )       )
              self.assertDictEqual(builder.info.features   ,datasets.Features({"""content""": datasets.Value("""string"""       )}       )       )
              lowerCamelCase__         =  builder.as_dataset()
              self.assertEqual(dset["""train"""].num_rows   ,_lowerCAmelCase       )
              self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples   ,_lowerCAmelCase       )
              # Order is not preserved when sharding, so we just check that all the elements are there
              self.assertListEqual(sorted(dset["""train"""]["""content"""]       )   ,sorted(["""foo""", """bar""", """foobar"""]       )       )
              self.assertTrue(
                  os.path.exists(os.path.join(_lowerCAmelCase   ,builder.name   ,"""default"""   ,"""0.0.0"""   ,"""dataset_info.json"""       )       )       )
              del dset






  @require_beam
  def 			UpperCamelCase_					(      self       ):

        with tempfile.TemporaryDirectory() as tmp_cache_dir:
              lowerCamelCase__         =  DummyBeamDataset(cache_dir=_lowerCAmelCase       )
              self.assertRaises(datasets.builder.MissingBeamOptions   ,builder.download_and_prepare       )






  @require_beam
  def 			UpperCamelCase_					(      self       ):
        lowerCamelCase__         =  len(get_test_nested_examples()       )
        with tempfile.TemporaryDirectory() as tmp_cache_dir:
              lowerCamelCase__         =  NestedBeamDataset(cache_dir=_lowerCAmelCase   ,beam_runner="""DirectRunner"""       )
              builder.download_and_prepare()
              self.assertTrue(
                  os.path.exists(
                      os.path.join(_lowerCAmelCase   ,builder.name   ,"""default"""   ,"""0.0.0"""   ,F'''{builder.name}-train.arrow'''       )       )       )
              self.assertDictEqual(
                  builder.info.features   ,datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""       )}       )}       )       )
              lowerCamelCase__         =  builder.as_dataset()
              self.assertEqual(dset["""train"""].num_rows   ,_lowerCAmelCase       )
              self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples   ,_lowerCAmelCase       )
              self.assertDictEqual(dset["""train"""][0]   ,get_test_nested_examples()[0][1]       )
              self.assertDictEqual(
                  dset["""train"""][expected_num_examples - 1]   ,get_test_nested_examples()[expected_num_examples - 1][1]       )
              self.assertTrue(
                  os.path.exists(os.path.join(_lowerCAmelCase   ,builder.name   ,"""default"""   ,"""0.0.0"""   ,"""dataset_info.json"""       )       )       )
              del dset









'''simple docstring'''







import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
							from PIL import Image

							from transformers import AutoProcessor, BertTokenizer, BlipImageProcessor, BlipProcessor, PreTrainedTokenizerFast




@require_vision
class 		UpperCamelCase__    (unittest.TestCase			):




		'''simple docstring'''





		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  tempfile.mkdtemp()

								lowerCamelCase__         =  BlipImageProcessor()
								lowerCamelCase__         =  BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-BertModel"""       )

								lowerCamelCase__         =  BlipProcessor(_lowerCAmelCase   ,_lowerCAmelCase       )

								processor.save_pretrained(self.tmpdirname       )






		def 			UpperCamelCase_					(      self   ,**_lowerCAmelCase       ):
								return AutoProcessor.from_pretrained(self.tmpdirname   ,**_lowerCAmelCase       ).tokenizer






		def 			UpperCamelCase_					(      self   ,**_lowerCAmelCase       ):
								return AutoProcessor.from_pretrained(self.tmpdirname   ,**_lowerCAmelCase       ).image_processor






		def 			UpperCamelCase_					(      self       ):
								shutil.rmtree(self.tmpdirname       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  [np.random.randint(2_55   ,size=(3, 30, 4_00)   ,dtype=np.uinta       )]

								lowerCamelCase__         =  [Image.fromarray(np.moveaxis(_lowerCAmelCase   ,0   ,-1       )       ) for x in image_inputs]

								return image_inputs






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  BlipProcessor(tokenizer=self.get_tokenizer()   ,image_processor=self.get_image_processor()       )
								processor.save_pretrained(self.tmpdirname       )

								lowerCamelCase__         =  self.get_tokenizer(bos_token="""(BOS)"""   ,eos_token="""(EOS)"""       )
								lowerCamelCase__         =  self.get_image_processor(do_normalize=_lowerCAmelCase   ,padding_value=1.0       )

								lowerCamelCase__         =  BlipProcessor.from_pretrained(
								    self.tmpdirname   ,bos_token="""(BOS)"""   ,eos_token="""(EOS)"""   ,do_normalize=_lowerCAmelCase   ,padding_value=1.0       )

								self.assertEqual(processor.tokenizer.get_vocab()   ,tokenizer_add_kwargs.get_vocab()       )
								self.assertIsInstance(processor.tokenizer   ,_lowerCAmelCase       )

								self.assertEqual(processor.image_processor.to_json_string()   ,image_processor_add_kwargs.to_json_string()       )
								self.assertIsInstance(processor.image_processor   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  self.prepare_image_inputs()

								lowerCamelCase__         =  image_processor(_lowerCAmelCase   ,return_tensors="""np"""       )
								lowerCamelCase__         =  processor(images=_lowerCAmelCase   ,return_tensors="""np"""       )

								for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()   ,input_processor[key].sum()   ,delta=1E-2       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  """lower newer"""

								lowerCamelCase__         =  processor(text=_lowerCAmelCase       )

								lowerCamelCase__         =  tokenizer(_lowerCAmelCase   ,return_token_type_ids=_lowerCAmelCase       )

								for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]   ,encoded_processor[key]       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  """lower newer"""
								lowerCamelCase__         =  self.prepare_image_inputs()

								lowerCamelCase__         =  processor(text=_lowerCAmelCase   ,images=_lowerCAmelCase       )

								self.assertListEqual(list(inputs.keys()       )   ,["""pixel_values""", """input_ids""", """attention_mask"""]       )

								# test if it raises when no input is passed
								with pytest.raises(_lowerCAmelCase       ):
														processor()






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

								lowerCamelCase__         =  processor.batch_decode(_lowerCAmelCase       )
								lowerCamelCase__         =  tokenizer.batch_decode(_lowerCAmelCase       )

								self.assertListEqual(_lowerCAmelCase   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  """lower newer"""
								lowerCamelCase__         =  self.prepare_image_inputs()

								lowerCamelCase__         =  processor(text=_lowerCAmelCase   ,images=_lowerCAmelCase       )

								# For now the processor supports only ['pixel_values', 'input_ids', 'attention_mask']
								self.assertListEqual(list(inputs.keys()       )   ,["""pixel_values""", """input_ids""", """attention_mask"""]       )

