





'''simple docstring'''







import inspect
import unittest

from transformers import YolosConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch
   from torch import nn

   from transformers import YolosForObjectDetection, YolosModel
   from transformers.models.yolos.modeling_yolos import YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
   from PIL import Image

   from transformers import AutoImageProcessor
class 			_lowercase						:







     def __init__( self    ,   A__    ,   A__=13    ,   A__=[30, 30]    ,   A__=2    ,   A__=3    ,   A__=True    ,   A__=True    ,   A__=32    ,   A__=5    ,   A__=4    ,   A__=37    ,   A__="gelu"    ,   A__=0.1    ,   A__=0.1    ,   A__=10    ,   A__=0.0_2    ,   A__=3    ,   A__=None    ,   A__=8    ,   A__=10    ,   ) ->	List[Any]:
      snake_case				=			parent
      snake_case				=			batch_size
      snake_case				=			image_size
      snake_case				=			patch_size
      snake_case				=			num_channels
      snake_case				=			is_training
      snake_case				=			use_labels
      snake_case				=			hidden_size
      snake_case				=			num_hidden_layers
      snake_case				=			num_attention_heads
      snake_case				=			intermediate_size
      snake_case				=			hidden_act
      snake_case				=			hidden_dropout_prob
      snake_case				=			attention_probs_dropout_prob
      snake_case				=			type_sequence_label_size
      snake_case				=			initializer_range
      snake_case				=			num_labels
      snake_case				=			scope
      snake_case				=			n_targets
      snake_case				=			num_detection_tokens
      # we set the expected sequence length (which is used in several tests)
      # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) + num_detection_tokens
      snake_case				=			(image_size[1] // patch_size) * (image_size[0] // patch_size)
      snake_case				=			num_patches + 1 + self.num_detection_tokens







     def     UpperCamelCase			( self    ) ->	Any:
      snake_case				=			floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]    )

      snake_case				=			None
      if self.use_labels:
       # labels is a list of Dict (each Dict being the labels for a given example in the batch)
       snake_case				=			[]
       for i in range(self.batch_size    ):
        snake_case				=			{}
        snake_case				=			torch.randint(
            high=self.num_labels    ,   size=(self.n_targets,)    ,   device=_SCREAMING_SNAKE_CASE    )
        snake_case				=			torch.rand(self.n_targets    ,   4    ,   device=_SCREAMING_SNAKE_CASE    )
        labels.append(_SCREAMING_SNAKE_CASE    )

      snake_case				=			self.get_config()

      return config, pixel_values, labels







     def     UpperCamelCase			( self    ) ->	List[Any]:
      return YolosConfig(
          image_size=self.image_size    ,   patch_size=self.patch_size    ,   num_channels=self.num_channels    ,   hidden_size=self.hidden_size    ,   num_hidden_layers=self.num_hidden_layers    ,   num_attention_heads=self.num_attention_heads    ,   intermediate_size=self.intermediate_size    ,   hidden_act=self.hidden_act    ,   hidden_dropout_prob=self.hidden_dropout_prob    ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,   is_decoder=_SCREAMING_SNAKE_CASE    ,   initializer_range=self.initializer_range    ,   num_detection_tokens=self.num_detection_tokens    ,   num_labels=self.num_labels    ,   )







     def     UpperCamelCase			( self    ,   A__    ,   A__    ,   A__    ) ->	Tuple:
      snake_case				=			YolosModel(config=_SCREAMING_SNAKE_CASE    )
      model.to(_SCREAMING_SNAKE_CASE    )
      model.eval()
      snake_case				=			model(_SCREAMING_SNAKE_CASE    )
      self.parent.assertEqual(
          result.last_hidden_state.shape    ,   (self.batch_size, self.expected_seq_len, self.hidden_size)    )







     def     UpperCamelCase			( self    ,   A__    ,   A__    ,   A__    ) ->	int:
      snake_case				=			YolosForObjectDetection(_SCREAMING_SNAKE_CASE    )
      model.to(_SCREAMING_SNAKE_CASE    )
      model.eval()

      snake_case				=			model(pixel_values=_SCREAMING_SNAKE_CASE    )
      snake_case				=			model(_SCREAMING_SNAKE_CASE    )

      self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.num_detection_tokens, self.num_labels + 1)    )
      self.parent.assertEqual(result.pred_boxes.shape    ,   (self.batch_size, self.num_detection_tokens, 4)    )

      snake_case				=			model(pixel_values=_SCREAMING_SNAKE_CASE    ,   labels=_SCREAMING_SNAKE_CASE    )

      self.parent.assertEqual(result.loss.shape    ,   ()    )
      self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.num_detection_tokens, self.num_labels + 1)    )
      self.parent.assertEqual(result.pred_boxes.shape    ,   (self.batch_size, self.num_detection_tokens, 4)    )







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				=			self.prepare_config_and_inputs()
      snake_case				,		snake_case				,		snake_case				=			config_and_inputs
      snake_case				=			{'''pixel_values''': pixel_values}
      return config, inputs_dict




@require_torch
class 			_lowercase						(       lowerCAmelCase__				,		lowerCAmelCase__				,		unittest.TestCase					):
     _UpperCAmelCase										=					(YolosModel, YolosForObjectDetection) if is_torch_available() else ()
     _UpperCAmelCase										=					(
         {"feature-extraction": YolosModel, "object-detection": YolosForObjectDetection} if is_torch_available() else {}
     )

     _UpperCAmelCase										=					False
     _UpperCAmelCase										=					False
     _UpperCAmelCase										=					False
     _UpperCAmelCase										=					False







     def     UpperCamelCase			( self    ,   A__    ,   A__    ,   A__=False    ) ->	Any:
      snake_case				=			super()._prepare_for_class(_SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    ,   return_labels=_SCREAMING_SNAKE_CASE    )

      if return_labels:
       if model_class.__name__ == "YolosForObjectDetection":
        snake_case				=			[]
        for i in range(self.model_tester.batch_size    ):
         snake_case				=			{}
         snake_case				=			torch.ones(
             size=(self.model_tester.n_targets,)    ,   device=_SCREAMING_SNAKE_CASE    ,   dtype=torch.long    )
         snake_case				=			torch.ones(
             self.model_tester.n_targets    ,   4    ,   device=_SCREAMING_SNAKE_CASE    ,   dtype=torch.float    )
         labels.append(_SCREAMING_SNAKE_CASE    )
        snake_case				=			labels

      return inputs_dict







     def     UpperCamelCase			( self    ) ->	List[str]:
      snake_case				=			YolosModelTester(self    )
      snake_case				=			ConfigTester(self    ,   config_class=_SCREAMING_SNAKE_CASE    ,   has_text_modality=_SCREAMING_SNAKE_CASE    ,   hidden_size=37    )







     def     UpperCamelCase			( self    ) ->	int:
      self.config_tester.run_common_tests()







     def     UpperCamelCase			( self    ) ->	Any:
      # YOLOS does not use inputs_embeds
      pass







     def     UpperCamelCase			( self    ) ->	Optional[Any]:
      snake_case				,		snake_case				=			self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       snake_case				=			model_class(_SCREAMING_SNAKE_CASE    )
       self.assertIsInstance(model.get_input_embeddings()    ,   (nn.Module)    )
       snake_case				=			model.get_output_embeddings()
       self.assertTrue(x is None or isinstance(_SCREAMING_SNAKE_CASE    ,   nn.Linear    )    )







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      snake_case				,		snake_case				=			self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       snake_case				=			model_class(_SCREAMING_SNAKE_CASE    )
       snake_case				=			inspect.signature(model.forward    )
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       snake_case				=			[*signature.parameters.keys()]

       snake_case				=			['''pixel_values''']
       self.assertListEqual(arg_names[:1]    ,   _SCREAMING_SNAKE_CASE    )







     def     UpperCamelCase			( self    ) ->	List[str]:
      snake_case				=			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE    )







     def     UpperCamelCase			( self    ) ->	Optional[int]:
      snake_case				,		snake_case				=			self.model_tester.prepare_config_and_inputs_for_common()
      snake_case				=			True

      # in YOLOS, the seq_len is different
      snake_case				=			self.model_tester.expected_seq_len
      for model_class in self.all_model_classes:
       snake_case				=			True
       snake_case				=			False
       snake_case				=			True
       snake_case				=			model_class(_SCREAMING_SNAKE_CASE    )
       model.to(_SCREAMING_SNAKE_CASE    )
       model.eval()
       with torch.no_grad():
        snake_case				=			model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    )    )
       snake_case				=			outputs.attentions
       self.assertEqual(len(_SCREAMING_SNAKE_CASE    )    ,   self.model_tester.num_hidden_layers    )

       # check that output_attentions also work using config
       del inputs_dict["output_attentions"]
       snake_case				=			True
       snake_case				=			model_class(_SCREAMING_SNAKE_CASE    )
       model.to(_SCREAMING_SNAKE_CASE    )
       model.eval()
       with torch.no_grad():
        snake_case				=			model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    )    )
       snake_case				=			outputs.attentions
       self.assertEqual(len(_SCREAMING_SNAKE_CASE    )    ,   self.model_tester.num_hidden_layers    )

       self.assertListEqual(
           list(attentions[0].shape[-3:]    )    ,   [self.model_tester.num_attention_heads, seq_len, seq_len]    ,   )
       snake_case				=			len(_SCREAMING_SNAKE_CASE    )

       # Check attention is always last and order is fine
       snake_case				=			True
       snake_case				=			True
       snake_case				=			model_class(_SCREAMING_SNAKE_CASE    )
       model.to(_SCREAMING_SNAKE_CASE    )
       model.eval()
       with torch.no_grad():
        snake_case				=			model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    )    )

       snake_case				=			1
       self.assertEqual(out_len + added_hidden_states    ,   len(_SCREAMING_SNAKE_CASE    )    )

       snake_case				=			outputs.attentions

       self.assertEqual(len(_SCREAMING_SNAKE_CASE    )    ,   self.model_tester.num_hidden_layers    )
       self.assertListEqual(
           list(self_attentions[0].shape[-3:]    )    ,   [self.model_tester.num_attention_heads, seq_len, seq_len]    ,   )







     def     UpperCamelCase			( self    ) ->	str:
      def check_hidden_states_output(A__    ,   A__    ,   A__    ):
       snake_case				=			model_class(_SCREAMING_SNAKE_CASE    )
       model.to(_SCREAMING_SNAKE_CASE    )
       model.eval()

       with torch.no_grad():
        snake_case				=			model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    )    )

       snake_case				=			outputs.hidden_states

       snake_case				=			getattr(
           self.model_tester    ,   '''expected_num_hidden_layers'''    ,   self.model_tester.num_hidden_layers + 1    )
       self.assertEqual(len(_SCREAMING_SNAKE_CASE    )    ,   _SCREAMING_SNAKE_CASE    )

       # YOLOS has a different seq_length
       snake_case				=			self.model_tester.expected_seq_len

       self.assertListEqual(
           list(hidden_states[0].shape[-2:]    )    ,   [seq_length, self.model_tester.hidden_size]    ,   )

      snake_case				,		snake_case				=			self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       snake_case				=			True
       check_hidden_states_output(_SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    )

       # check that output_hidden_states also work using config
       del inputs_dict["output_hidden_states"]
       snake_case				=			True

       check_hidden_states_output(_SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    ,   _SCREAMING_SNAKE_CASE    )







     def     UpperCamelCase			( self    ) ->	str:
      snake_case				=			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_object_detection(*_SCREAMING_SNAKE_CASE    )







     @slow
     def     UpperCamelCase			( self    ) ->	List[str]:
      for model_name in YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       snake_case				=			YolosModel.from_pretrained(_SCREAMING_SNAKE_CASE    )
       self.assertIsNotNone(_SCREAMING_SNAKE_CASE    )




def    __UpperCamelCase	(	) ->List[Any]:
 snake_case				=			Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''     )
 return image






@require_torch
@require_vision
class 			_lowercase						(       unittest.TestCase					):







     @cached_property
     def     UpperCamelCase			( self    ) ->	int:
      return AutoImageProcessor.from_pretrained('''hustvl/yolos-small'''    ) if is_vision_available() else None







     @slow
     def     UpperCamelCase			( self    ) ->	Any:
      snake_case				=			YolosForObjectDetection.from_pretrained('''hustvl/yolos-small'''    ).to(_SCREAMING_SNAKE_CASE    )

      snake_case				=			self.default_image_processor
      snake_case				=			prepare_img()
      snake_case				=			image_processor(images=_SCREAMING_SNAKE_CASE    ,   return_tensors='''pt'''    ).to(_SCREAMING_SNAKE_CASE    )

      # forward pass
      with torch.no_grad():
       snake_case				=			model(inputs.pixel_values    )

      # verify outputs
      snake_case				=			torch.Size((1, 1_00, 92)    )
      self.assertEqual(outputs.logits.shape    ,   _SCREAMING_SNAKE_CASE    )

      snake_case				=			torch.tensor(
          [[-24.02_48, -10.30_24, -14.82_90], [-42.03_92, -16.82_00, -27.43_34], [-27.27_43, -11.81_54, -18.71_48]]    ,   device=_SCREAMING_SNAKE_CASE    ,   )
      snake_case				=			torch.tensor(
          [[0.2_5_5_9, 0.5_4_5_5, 0.4_7_0_6], [0.2_9_8_9, 0.7_2_7_9, 0.1_8_7_5], [0.7_7_3_2, 0.4_0_1_7, 0.4_4_6_2]]    ,   device=_SCREAMING_SNAKE_CASE    )
      self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]    ,   _SCREAMING_SNAKE_CASE    ,   atol=1e-4    )    )
      self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3]    ,   _SCREAMING_SNAKE_CASE    ,   atol=1e-4    )    )

      # verify postprocessing
      snake_case				=			image_processor.post_process_object_detection(
          _SCREAMING_SNAKE_CASE    ,   threshold=0.3    ,   target_sizes=[image.size[::-1]]    )[0]
      snake_case				=			torch.tensor([0.9_9_9_4, 0.9_7_9_0, 0.9_9_6_4, 0.9_9_7_2, 0.9_8_6_1]    ).to(_SCREAMING_SNAKE_CASE    )
      snake_case				=			[75, 75, 17, 63, 17]
      snake_case				=			torch.tensor([3_35.06_09, 79.38_48, 3_75.42_16, 1_87.24_95]    ).to(_SCREAMING_SNAKE_CASE    )

      self.assertEqual(len(results['''scores''']    )    ,   5    )
      self.assertTrue(torch.allclose(results['''scores''']    ,   _SCREAMING_SNAKE_CASE    ,   atol=1e-4    )    )
      self.assertSequenceEqual(results['''labels'''].tolist()    ,   _SCREAMING_SNAKE_CASE    )
      self.assertTrue(torch.allclose(results['''boxes'''][0, :]    ,   _SCREAMING_SNAKE_CASE    )    )








'''simple docstring'''







import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


_lowercase       						=			[
    {'dataset': 'wikipedia', 'config_name': '20220301.de'},
    {'dataset': 'wikipedia', 'config_name': '20220301.en'},
    {'dataset': 'wikipedia', 'config_name': '20220301.fr'},
    {'dataset': 'wikipedia', 'config_name': '20220301.frr'},
    {'dataset': 'wikipedia', 'config_name': '20220301.it'},
    {'dataset': 'wikipedia', 'config_name': '20220301.simple'},
    {'dataset': 'snli', 'config_name': 'plain_text'},
    {'dataset': 'eli5', 'config_name': 'LFQA_reddit'},
    {'dataset': 'wiki40b', 'config_name': 'en'},
    {'dataset': 'wiki_dpr', 'config_name': 'psgs_w100.nq.compressed'},
    {'dataset': 'wiki_dpr', 'config_name': 'psgs_w100.nq.no_index'},
    {'dataset': 'wiki_dpr', 'config_name': 'psgs_w100.multiset.no_index'},
    {'dataset': 'natural_questions', 'config_name': 'default'},
]

def    __UpperCamelCase	(	a    : Dict=True     ) ->str:
 if with_config:
  return [
      {
          "testcase_name": d["dataset"] + "/" + d["config_name"],
          "dataset": d["dataset"],
          "config_name": d["config_name"],
      }
      for d in DATASETS_ON_HF_GCP
  ]
 else:
  return [
      {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
  ]
@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=__a					)					)
class 			_lowercase						(       __a					):
     _UpperCAmelCase										=					None
     _UpperCAmelCase										=					None







     def     UpperCamelCase			( self    ,   A__    ,   A__    ) ->	str:
      with TemporaryDirectory() as tmp_dir:
       snake_case				=			dataset_module_factory(A__    ,   cache_dir=A__    )

       snake_case				=			import_main_class(dataset_module.module_path    ,   dataset=A__    )

       snake_case				=			builder_cls(
           cache_dir=A__    ,   config_name=A__    ,   hash=dataset_module.hash    ,   )

       snake_case				=			'''/'''.join(
           [
               HF_GCP_BASE_URL,
               builder_instance._relative_data_dir(with_hash=A__    ).replace(os.sep    ,   '''/'''    ),
               config.DATASET_INFO_FILENAME,
           ]    )
       snake_case				=			cached_path(A__    ,   cache_dir=A__    )
       self.assertTrue(os.path.exists(A__    )    )




@pytest.mark.integration
def    __UpperCamelCase	(	a    : List[str]     ) ->Any:
 snake_case				=			tmp_path_factory.mktemp('''test_hf_gcp'''     ) / '''test_wikipedia_simple'''
 snake_case				=			dataset_module_factory('''wikipedia'''					,   cache_dir=a     )
 snake_case				=			import_main_class(dataset_module.module_path     )
 snake_case				=			builder_cls(
     cache_dir=a					,   config_name='''20220301.frr'''					,   hash=dataset_module.hash					,   )
 # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
 snake_case				=			None
 builder_instance.download_and_prepare()
 snake_case				=			builder_instance.as_dataset()
 assert ds






@pytest.mark.integration
def    __UpperCamelCase	(	a    : Any     ) ->Union[str, Any]:
 snake_case				=			dataset_module_factory('''wikipedia'''					,   cache_dir=a     )
 snake_case				=			import_main_class(dataset_module.module_path					,   dataset=a     )
 snake_case				=			builder_cls(
     cache_dir=a					,   config_name='''20220301.frr'''					,   hash=dataset_module.hash					,   )
 snake_case				=			builder_instance.as_streaming_dataset()
 assert ds
 assert isinstance(a					,   a     )
 assert "train" in ds
 assert isinstance(ds['''train''']					,   a     )
 assert next(iter(ds['''train''']     )     )

