
'''simple docstring'''







import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
      from sagemaker import Session, TrainingJobAnalytics
      from sagemaker.huggingface import HuggingFace


@pytest.mark.skipif(
    literal_eval(os.getenv("""TEST_SAGEMAKER"""		,     """False"""			)			) is not True		,     reason="""Skipping test because should only be run when releasing minor transformers version"""		,     )
@pytest.mark.usefixtures("""sm_env"""			)
@parameterized_class(
    [
        {
            """framework""": """pytorch""",
            """script""": """run_glue_model_parallelism.py""",
            """model_name_or_path""": """roberta-large""",
            """instance_type""": """ml.p3dn.24xlarge""",
            """results""": {"""train_runtime""": 1_600, """eval_accuracy""": 0.3, """eval_loss""": 1.2},
        },
        {
            """framework""": """pytorch""",
            """script""": """run_glue.py""",
            """model_name_or_path""": """roberta-large""",
            """instance_type""": """ml.p3dn.24xlarge""",
            """results""": {"""train_runtime""": 1_600, """eval_accuracy""": 0.3, """eval_loss""": 1.2},
        },
    ]			)
class 			lowerCAmelCase (				unittest.TestCase			):




  def     lowercase       (				self      ):
        if self.framework == "pytorch":
              subprocess.run(
                  F'cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'.split()					,						encoding="utf-8"					,						check=UpperCamelCase					,						)
        assert hasattr(self					,						"env"      )




  def     lowercase       (				self					,						UpperCamelCase      ):
        # configuration for running training on smdistributed Model Parallel
        _SCREAMING_SNAKE_CASE				= {
            "enabled": True,
            "processes_per_host": 8,
        }
        _SCREAMING_SNAKE_CASE				= {
            "enabled": True,
            "parameters": {
                "microbatches": 4,
                "placement_strategy": "spread",
                "pipeline": "interleaved",
                "optimize": "speed",
                "partitions": 4,
                "ddp": True,
            },
        }

        _SCREAMING_SNAKE_CASE				= {"smdistributed": {"modelparallel": smp_options}, "mpi": mpi_options}

        _SCREAMING_SNAKE_CASE				= "trainer" if self.script == "run_glue.py" else "smtrainer"
        # creates estimator
        return HuggingFace(
            entry_point=self.script					,						source_dir=self.env.test_path					,						role=self.env.role					,						image_uri=self.env.image_uri					,						base_job_name=F'{self.env.base_job_name}-{instance_count}-smp-{name_extension}'					,						instance_count=UpperCamelCase					,						instance_type=self.instance_type					,						debugger_hook_config=UpperCamelCase					,						hyperparameters={
                **self.env.hyperparameters,
                "model_name_or_path": self.model_name_or_path,
                "max_steps": 500,
            }					,						metric_definitions=self.env.metric_definitions					,						distribution=UpperCamelCase					,						py_version="py36"					,						)




  def     lowercase       (				self					,						UpperCamelCase      ):
        TrainingJobAnalytics(UpperCamelCase      ).export_csv(F'{self.env.test_path}/{job_name}_metrics.csv'      )




  @parameterized.expand([(1,)]      )
  def     lowercase       (				self					,						UpperCamelCase      ):
        # create estimator
        _SCREAMING_SNAKE_CASE				= self.create_estimator(UpperCamelCase      )

        # run training
        estimator.fit()

        # result dataframe
        _SCREAMING_SNAKE_CASE				= TrainingJobAnalytics(estimator.latest_training_job.name      ).dataframe()

        # extract kpis
        _SCREAMING_SNAKE_CASE				= list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]      )
        _SCREAMING_SNAKE_CASE				= list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]      )
        # get train time from SageMaker job, this includes starting, preprocessing, stopping
        _SCREAMING_SNAKE_CASE				= (
            Session().describe_training_job(estimator.latest_training_job.name      ).get("TrainingTimeInSeconds"					,						999_999      )
        )

        # assert kpis
        assert train_runtime <= self.results["train_runtime"]
        assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy      )
        assert all(t <= self.results["eval_loss"] for t in eval_loss      )

        # dump tests result into json file to share in PR
        with open(F'{estimator.latest_training_job.name}.json'					,						"w"      ) as outfile:
              json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}					,						UpperCamelCase      )

'''simple docstring'''







from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
						import torch

if is_tf_available():
						import tensorflow as tf


class 			lowerCAmelCase (				__UpperCAmelCase			):
		a							:					Optional[int]       =					["""image_processor"""]
		a							:					Tuple       =					"""SamImageProcessor"""




		def __init__(				self					,						UpperCamelCase      ):
								super().__init__(UpperCamelCase      )
								_SCREAMING_SNAKE_CASE				= self.image_processor
								_SCREAMING_SNAKE_CASE				= -10
								_SCREAMING_SNAKE_CASE				= self.image_processor.size["longest_edge"]




		def __call__(				self					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase = None					,						**UpperCamelCase					,						):
								_SCREAMING_SNAKE_CASE				= self.image_processor(
								    UpperCamelCase					,						return_tensors=UpperCamelCase					,						**UpperCamelCase					,						)

								# pop arguments that are not used in the foward but used nevertheless
								_SCREAMING_SNAKE_CASE				= encoding_image_processor["original_sizes"]

								if hasattr(UpperCamelCase					,						"numpy"      ):  # Checks if Torch or TF tensor
														_SCREAMING_SNAKE_CASE				= original_sizes.numpy()

								_SCREAMING_SNAKE_CASE,							_SCREAMING_SNAKE_CASE,							_SCREAMING_SNAKE_CASE				= self._check_and_preprocess_points(
								    input_points=UpperCamelCase					,						input_labels=UpperCamelCase					,						input_boxes=UpperCamelCase					,						)

								_SCREAMING_SNAKE_CASE				= self._normalize_and_convert(
								    UpperCamelCase					,						UpperCamelCase					,						input_points=UpperCamelCase					,						input_labels=UpperCamelCase					,						input_boxes=UpperCamelCase					,						return_tensors=UpperCamelCase					,						)

								return encoding_image_processor




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase="pt"					,						):
								if input_points is not None:
														if len(UpperCamelCase      ) != len(UpperCamelCase      ):
																				_SCREAMING_SNAKE_CASE				= [
																				    self._normalize_coordinates(self.target_size					,						UpperCamelCase					,						original_sizes[0]      ) for point in input_points
																				]
														else:
																				_SCREAMING_SNAKE_CASE				= [
																				    self._normalize_coordinates(self.target_size					,						UpperCamelCase					,						UpperCamelCase      )
																				    for point, original_size in zip(UpperCamelCase					,						UpperCamelCase      )
																				]
														# check that all arrays have the same shape
														if not all(point.shape == input_points[0].shape for point in input_points      ):
																				if input_labels is not None:
																										_SCREAMING_SNAKE_CASE,							_SCREAMING_SNAKE_CASE				= self._pad_points_and_labels(UpperCamelCase					,						UpperCamelCase      )

														_SCREAMING_SNAKE_CASE				= np.array(UpperCamelCase      )

								if input_labels is not None:
														_SCREAMING_SNAKE_CASE				= np.array(UpperCamelCase      )

								if input_boxes is not None:
														if len(UpperCamelCase      ) != len(UpperCamelCase      ):
																				_SCREAMING_SNAKE_CASE				= [
																				    self._normalize_coordinates(self.target_size					,						UpperCamelCase					,						original_sizes[0]					,						is_bounding_box=UpperCamelCase      )
																				    for box in input_boxes
																				]
														else:
																				_SCREAMING_SNAKE_CASE				= [
																				    self._normalize_coordinates(self.target_size					,						UpperCamelCase					,						UpperCamelCase					,						is_bounding_box=UpperCamelCase      )
																				    for box, original_size in zip(UpperCamelCase					,						UpperCamelCase      )
																				]
														_SCREAMING_SNAKE_CASE				= np.array(UpperCamelCase      )

								if input_boxes is not None:
														if return_tensors == "pt":
																				_SCREAMING_SNAKE_CASE				= torch.from_numpy(UpperCamelCase      )
																				# boxes batch size of 1 by default
																				_SCREAMING_SNAKE_CASE				= input_boxes.unsqueeze(1      ) if len(input_boxes.shape      ) != 3 else input_boxes
														elif return_tensors == "tf":
																				_SCREAMING_SNAKE_CASE				= tf.convert_to_tensor(UpperCamelCase      )
																				# boxes batch size of 1 by default
																				_SCREAMING_SNAKE_CASE				= tf.expand_dims(UpperCamelCase					,						1      ) if len(input_boxes.shape      ) != 3 else input_boxes
														encoding_image_processor.update({"input_boxes": input_boxes}      )
								if input_points is not None:
														if return_tensors == "pt":
																				_SCREAMING_SNAKE_CASE				= torch.from_numpy(UpperCamelCase      )
																				# point batch size of 1 by default
																				_SCREAMING_SNAKE_CASE				= input_points.unsqueeze(1      ) if len(input_points.shape      ) != 4 else input_points
														elif return_tensors == "tf":
																				_SCREAMING_SNAKE_CASE				= tf.convert_to_tensor(UpperCamelCase      )
																				# point batch size of 1 by default
																				_SCREAMING_SNAKE_CASE				= tf.expand_dims(UpperCamelCase					,						1      ) if len(input_points.shape      ) != 4 else input_points
														encoding_image_processor.update({"input_points": input_points}      )
								if input_labels is not None:
														if return_tensors == "pt":
																				_SCREAMING_SNAKE_CASE				= torch.from_numpy(UpperCamelCase      )
																				# point batch size of 1 by default
																				_SCREAMING_SNAKE_CASE				= input_labels.unsqueeze(1      ) if len(input_labels.shape      ) != 3 else input_labels
														elif return_tensors == "tf":
																				_SCREAMING_SNAKE_CASE				= tf.convert_to_tensor(UpperCamelCase      )
																				# point batch size of 1 by default
																				_SCREAMING_SNAKE_CASE				= tf.expand_dims(UpperCamelCase					,						1      ) if len(input_labels.shape      ) != 3 else input_labels
														encoding_image_processor.update({"input_labels": input_labels}      )

								return encoding_image_processor




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= max([point.shape[0] for point in input_points]      )
								_SCREAMING_SNAKE_CASE				= []
								for i, point in enumerate(UpperCamelCase      ):
														if point.shape[0] != expected_nb_points:
																				_SCREAMING_SNAKE_CASE				= np.concatenate(
																				    [point, np.zeros((expected_nb_points - point.shape[0], 2)      ) + self.point_pad_value]					,						axis=0      )
																				_SCREAMING_SNAKE_CASE				= np.append(input_labels[i]					,						[self.point_pad_value]      )
														processed_input_points.append(UpperCamelCase      )
								_SCREAMING_SNAKE_CASE				= processed_input_points
								return input_points, input_labels




		def     lowercase       (				self					,						UpperCamelCase					,						UpperCamelCase					,						UpperCamelCase					,						UpperCamelCase=False      ):
								_SCREAMING_SNAKE_CASE,							_SCREAMING_SNAKE_CASE				= original_size
								_SCREAMING_SNAKE_CASE,							_SCREAMING_SNAKE_CASE				= self.image_processor._get_preprocess_shape(UpperCamelCase					,						longest_edge=UpperCamelCase      )
								_SCREAMING_SNAKE_CASE				= deepcopy(UpperCamelCase      ).astype(UpperCamelCase      )

								if is_bounding_box:
														_SCREAMING_SNAKE_CASE				= coords.reshape(-1					,						2					,						2      )

								_SCREAMING_SNAKE_CASE				= coords[..., 0] * (new_w / old_w)
								_SCREAMING_SNAKE_CASE				= coords[..., 1] * (new_h / old_h)

								if is_bounding_box:
														_SCREAMING_SNAKE_CASE				= coords.reshape(-1					,						4      )

								return coords




		def     lowercase       (				self					,						UpperCamelCase=None					,						UpperCamelCase=None					,						UpperCamelCase=None					,						):
								if input_points is not None:
														if hasattr(UpperCamelCase					,						"numpy"      ):  # Checks for TF or Torch tensor
																				_SCREAMING_SNAKE_CASE				= input_points.numpy().tolist()

														if not isinstance(UpperCamelCase					,						UpperCamelCase      ) or not isinstance(input_points[0]					,						UpperCamelCase      ):
																				raise ValueError("Input points must be a list of list of floating points."      )
														_SCREAMING_SNAKE_CASE				= [np.array(UpperCamelCase      ) for input_point in input_points]
								else:
														_SCREAMING_SNAKE_CASE				= None

								if input_labels is not None:
														if hasattr(UpperCamelCase					,						"numpy"      ):
																				_SCREAMING_SNAKE_CASE				= input_labels.numpy().tolist()

														if not isinstance(UpperCamelCase					,						UpperCamelCase      ) or not isinstance(input_labels[0]					,						UpperCamelCase      ):
																				raise ValueError("Input labels must be a list of list integers."      )
														_SCREAMING_SNAKE_CASE				= [np.array(UpperCamelCase      ) for label in input_labels]
								else:
														_SCREAMING_SNAKE_CASE				= None

								if input_boxes is not None:
														if hasattr(UpperCamelCase					,						"numpy"      ):
																				_SCREAMING_SNAKE_CASE				= input_boxes.numpy().tolist()

														if (
														    not isinstance(UpperCamelCase					,						UpperCamelCase      )
														    or not isinstance(input_boxes[0]					,						UpperCamelCase      )
														    or not isinstance(input_boxes[0][0]					,						UpperCamelCase      )
														):
																				raise ValueError("Input boxes must be a list of list of list of floating points."      )
														_SCREAMING_SNAKE_CASE				= [np.array(UpperCamelCase      ).astype(np.floataa      ) for box in input_boxes]
								else:
														_SCREAMING_SNAKE_CASE				= None

								return input_points, input_labels, input_boxes




		@property
		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= self.image_processor.model_input_names
								return list(dict.fromkeys(UpperCamelCase      )      )




		def     lowercase       (				self					,						*UpperCamelCase					,						**UpperCamelCase      ):
								return self.image_processor.post_process_masks(*UpperCamelCase					,						**UpperCamelCase      )