



# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
      import botoa  # noqa: F401






def 	_lowerCAmelCase						(				__lowerCamelCase   :     Optional[int]     ):



 """simple docstring"""





 __SCREAMING_SNAKE_CASE       :    Optional[int]       =	botoa.client("iam"     )

 __SCREAMING_SNAKE_CASE       :    List[Any]       =	{
     "Version": "2012-10-17",
     "Statement": [
         {"Effect": "Allow", "Principal": {"Service": "sagemaker.amazonaws.com"}, "Action": "sts:AssumeRole"}
     ],
 }
 try:
  # create the role, associated with the chosen trust policy
  iam_client.create_role(
      RoleName=__lowerCamelCase		,	AssumeRolePolicyDocument=json.dumps(__lowerCamelCase		,	indent=2     )     )
  __SCREAMING_SNAKE_CASE       :    str       =	{
      "Version": "2012-10-17",
      "Statement": [
          {
              "Effect": "Allow",
              "Action": [
                  "sagemaker:*",
                  "ecr:GetDownloadUrlForLayer",
                  "ecr:BatchGetImage",
                  "ecr:BatchCheckLayerAvailability",
                  "ecr:GetAuthorizationToken",
                  "cloudwatch:PutMetricData",
                  "cloudwatch:GetMetricData",
                  "cloudwatch:GetMetricStatistics",
                  "cloudwatch:ListMetrics",
                  "logs:CreateLogGroup",
                  "logs:CreateLogStream",
                  "logs:DescribeLogStreams",
                  "logs:PutLogEvents",
                  "logs:GetLogEvents",
                  "s3:CreateBucket",
                  "s3:ListBucket",
                  "s3:GetBucketLocation",
                  "s3:GetObject",
                  "s3:PutObject",
              ],
              "Resource": "*",
          }
      ],
  }
  # attach policy to role
  iam_client.put_role_policy(
      RoleName=__lowerCamelCase		,	PolicyName=F"""{role_name}_policy_permission"""		,	PolicyDocument=json.dumps(__lowerCamelCase		,	indent=2     )		,	)
 except iam_client.exceptions.EntityAlreadyExistsException:
  print(F"""role {role_name} already exists. Using existing one"""     )






def 	_lowerCAmelCase						(				__lowerCamelCase   :     Dict     ):



 """simple docstring"""





 __SCREAMING_SNAKE_CASE       :    List[str]       =	botoa.client("iam"     )
 return iam_client.get_role(RoleName=__lowerCamelCase     )["Role"]["Arn"]






def 	_lowerCAmelCase						(				):



 """simple docstring"""





 __SCREAMING_SNAKE_CASE       :    Any       =	_ask_options(
     "How do you want to authorize?"		,	["AWS Profile", "Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) "]		,	__lowerCamelCase		,	)
 __SCREAMING_SNAKE_CASE       :    str       =	None
 if credentials_configuration == 0:
  __SCREAMING_SNAKE_CASE       :    List[str]       =	_ask_field("Enter your AWS Profile name: [default] "		,	default="default"     )
  __SCREAMING_SNAKE_CASE       :    Any       =	aws_profile
 else:
  print(
      "Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"
      "`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`"     )
  __SCREAMING_SNAKE_CASE       :    Dict       =	_ask_field("AWS Access Key ID: "     )
  __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	aws_access_key_id

  __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	_ask_field("AWS Secret Access Key: "     )
  __SCREAMING_SNAKE_CASE       :    Optional[int]       =	aws_secret_access_key

 __SCREAMING_SNAKE_CASE       :    List[Any]       =	_ask_field("Enter your AWS Region: [us-east-1]"		,	default="us-east-1"     )
 __SCREAMING_SNAKE_CASE       :    Any       =	aws_region

 __SCREAMING_SNAKE_CASE       :    int       =	_ask_options(
     "Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"		,	["Provide IAM Role name", "Create new IAM role using credentials"]		,	__lowerCamelCase		,	)
 if role_management == 0:
  __SCREAMING_SNAKE_CASE       :    List[str]       =	_ask_field("Enter your IAM role name: "     )
 else:
  __SCREAMING_SNAKE_CASE       :    Any       =	"accelerate_sagemaker_execution_role"
  print(F"""Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials"""     )
  _create_iam_role_for_sagemaker(__lowerCamelCase     )

 __SCREAMING_SNAKE_CASE       :    Any       =	_ask_field(
     "Do you want to use custom Docker image? [yes/NO]: "		,	_convert_yes_no_to_bool		,	default=__lowerCamelCase		,	error_message="Please enter yes or no."		,	)
 __SCREAMING_SNAKE_CASE       :    Tuple       =	None
 if is_custom_docker_image:
  __SCREAMING_SNAKE_CASE       :    List[Any]       =	_ask_field("Enter your Docker image: "		,	lambda __lowerCamelCase     : str(__lowerCamelCase     ).lower()     )

 __SCREAMING_SNAKE_CASE       :    Dict       =	_ask_field(
     "Do you want to provide SageMaker input channels with data locations? [yes/NO]: "		,	_convert_yes_no_to_bool		,	default=__lowerCamelCase		,	error_message="Please enter yes or no."		,	)
 __SCREAMING_SNAKE_CASE       :    List[Any]       =	None
 if is_sagemaker_inputs_enabled:
  __SCREAMING_SNAKE_CASE       :    List[str]       =	_ask_field(
      "Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): "		,	lambda __lowerCamelCase     : str(__lowerCamelCase     ).lower()		,	)

 __SCREAMING_SNAKE_CASE       :    Dict       =	_ask_field(
     "Do you want to enable SageMaker metrics? [yes/NO]: "		,	_convert_yes_no_to_bool		,	default=__lowerCamelCase		,	error_message="Please enter yes or no."		,	)
 __SCREAMING_SNAKE_CASE       :    int       =	None
 if is_sagemaker_metrics_enabled:
  __SCREAMING_SNAKE_CASE       :    Dict       =	_ask_field(
      "Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): "		,	lambda __lowerCamelCase     : str(__lowerCamelCase     ).lower()		,	)

 __SCREAMING_SNAKE_CASE       :    Any       =	_ask_options(
     "What is the distributed mode?"		,	["No distributed training", "Data parallelism"]		,	_convert_sagemaker_distributed_mode		,	)
 __SCREAMING_SNAKE_CASE       :    Tuple       =	{}
 __SCREAMING_SNAKE_CASE       :    str       =	_ask_field(
     "Do you wish to optimize your script with torch dynamo?[yes/NO]:"		,	_convert_yes_no_to_bool		,	default=__lowerCamelCase		,	error_message="Please enter yes or no."		,	)
 if use_dynamo:
  __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	"dynamo_"
  __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	_ask_options(
      "Which dynamo backend would you like to use?"		,	[x.lower() for x in DYNAMO_BACKENDS]		,	_convert_dynamo_backend		,	default=2		,	)
  __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	_ask_field(
      "Do you want to customize the defaults sent to torch.compile? [yes/NO]: "		,	_convert_yes_no_to_bool		,	default=__lowerCamelCase		,	error_message="Please enter yes or no."		,	)

  if use_custom_options:
   __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	_ask_options(
       "Which mode do you want to use?"		,	__lowerCamelCase		,	lambda __lowerCamelCase     : TORCH_DYNAMO_MODES[int(__lowerCamelCase     )]		,	default="default"		,	)
   __SCREAMING_SNAKE_CASE       :    str       =	_ask_field(
       "Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: "		,	_convert_yes_no_to_bool		,	default=__lowerCamelCase		,	error_message="Please enter yes or no."		,	)
   __SCREAMING_SNAKE_CASE       :    Any       =	_ask_field(
       "Do you want to enable dynamic shape tracing? [yes/NO]: "		,	_convert_yes_no_to_bool		,	default=__lowerCamelCase		,	error_message="Please enter yes or no."		,	)
 __SCREAMING_SNAKE_CASE       :    Optional[int]       =	"Which EC2 instance type you want to use for your training?"
 if distributed_type != SageMakerDistributedType.NO:
  __SCREAMING_SNAKE_CASE       :    List[Any]       =	_ask_options(
      __lowerCamelCase		,	__lowerCamelCase		,	lambda __lowerCamelCase     : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(__lowerCamelCase     )]     )
 else:
  eca_instance_query += "? [ml.p3.2xlarge]:"
  __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	_ask_field(__lowerCamelCase		,	lambda __lowerCamelCase     : str(__lowerCamelCase     ).lower()		,	default="ml.p3.2xlarge"     )

 __SCREAMING_SNAKE_CASE       :    List[Any]       =	1
 if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
  __SCREAMING_SNAKE_CASE       :    Any       =	_ask_field(
      "How many machines do you want use? [1]: "		,	__lowerCamelCase		,	default=1		,	)

 __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	_ask_options(
     "Do you wish to use FP16 or BF16 (mixed precision)?"		,	["no", "fp16", "bf16", "fp8"]		,	_convert_mixed_precision		,	)

 if use_dynamo and mixed_precision == "no":
  print(
      "Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts."     )

 return SageMakerConfig(
     image_uri=__lowerCamelCase		,	compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER		,	distributed_type=__lowerCamelCase		,	use_cpu=__lowerCamelCase		,	dynamo_config=__lowerCamelCase		,	eca_instance_type=__lowerCamelCase		,	profile=__lowerCamelCase		,	region=__lowerCamelCase		,	iam_role_name=__lowerCamelCase		,	mixed_precision=__lowerCamelCase		,	num_machines=__lowerCamelCase		,	sagemaker_inputs_file=__lowerCamelCase		,	sagemaker_metrics_file=__lowerCamelCase		,	)











import unittest
from typing import Dict, List, Optional, Union

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import BridgeTowerImageProcessor

class       _SCREAMING_SNAKE_CASE	(unittest.TestCase ):





     def __init__(       self :							Union[str, Any]     ,  UpperCamelCase :							List[Any]     ,  UpperCamelCase :							bool = True     ,  UpperCamelCase :							Dict[str, int] = None     ,  UpperCamelCase :							int = 3_2     ,  UpperCamelCase :							bool = True     ,  UpperCamelCase :							Union[int, float] = 1 / 2_5_5     ,  UpperCamelCase :							bool = True     ,  UpperCamelCase :							bool = True     ,  UpperCamelCase :							Optional[Union[float, List[float]]] = [0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3]     ,  UpperCamelCase :							Optional[Union[float, List[float]]] = [0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1]     ,  UpperCamelCase :							bool = True     ,  UpperCamelCase :							Dict=7     ,  UpperCamelCase :							Dict=3_0     ,  UpperCamelCase :							Tuple=4_0_0     ,  UpperCamelCase :							Union[str, Any]=3     ,  )->Optional[int]:
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	parent
      __SCREAMING_SNAKE_CASE       :    str       =	do_resize
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	size if size is not None else {"shortest_edge": 2_8_8}
      __SCREAMING_SNAKE_CASE       :    List[str]       =	size_divisor
      __SCREAMING_SNAKE_CASE       :    str       =	do_rescale
      __SCREAMING_SNAKE_CASE       :    int       =	rescale_factor
      __SCREAMING_SNAKE_CASE       :    Tuple       =	do_normalize
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	do_center_crop
      __SCREAMING_SNAKE_CASE       :    List[str]       =	image_mean
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	image_std
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	do_pad
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	batch_size
      __SCREAMING_SNAKE_CASE       :    str       =	num_channels
      __SCREAMING_SNAKE_CASE       :    List[str]       =	min_resolution
      __SCREAMING_SNAKE_CASE       :    Tuple       =	max_resolution





     def 	__snake_case    (       self :							Optional[Any]				)->List[str]:
      return {
          "image_mean": self.image_mean,
          "image_std": self.image_std,
          "do_normalize": self.do_normalize,
          "do_resize": self.do_resize,
          "size": self.size,
          "size_divisor": self.size_divisor,
      }






     def 	__snake_case    (       self :							Optional[Any]     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							Dict=False				)->Optional[Any]:
      if not batched:
       __SCREAMING_SNAKE_CASE       :    Optional[int]       =	self.size["shortest_edge"]
       __SCREAMING_SNAKE_CASE       :    List[Any]       =	image_inputs[0]
       if isinstance(UpperCamelCase     ,  Image.Image				):
        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    List[Any]       =	image.size
       else:
        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	image.shape[1], image.shape[2]
       __SCREAMING_SNAKE_CASE       :    Tuple       =	size / min(UpperCamelCase     ,  UpperCamelCase				)
       if h < w:
        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    int       =	size, scale * w
       else:
        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    List[str]       =	scale * h, size

       __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	int((1_3_3_3 / 8_0_0) * size				)
       if max(UpperCamelCase     ,  UpperCamelCase				) > max_size:
        __SCREAMING_SNAKE_CASE       :    Any       =	max_size / max(UpperCamelCase     ,  UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    List[Any]       =	newh * scale
        __SCREAMING_SNAKE_CASE       :    Optional[int]       =	neww * scale

       __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    List[Any]       =	int(newh + 0.5				), int(neww + 0.5				)
       __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Optional[Any]       =	(
           newh // self.size_divisor * self.size_divisor,
           neww // self.size_divisor * self.size_divisor,
       )

      else:
       __SCREAMING_SNAKE_CASE       :    Optional[int]       =	[]
       for image in image_inputs:
        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Optional[Any]       =	self.get_expected_values([image]				)
        expected_values.append((expected_height, expected_width)				)
       __SCREAMING_SNAKE_CASE       :    int       =	max(UpperCamelCase     ,  key=lambda UpperCamelCase				: item[0]				)[0]
       __SCREAMING_SNAKE_CASE       :    List[str]       =	max(UpperCamelCase     ,  key=lambda UpperCamelCase				: item[1]				)[1]

      return expected_height, expected_width






@require_torch
@require_vision
class       _SCREAMING_SNAKE_CASE	(UpperCamelCase			,    unittest.TestCase ):
     lowerCAmelCase					=			BridgeTowerImageProcessor if is_vision_available() else None





     def 	__snake_case    (       self :							str				)->int:
      __SCREAMING_SNAKE_CASE       :    str       =	BridgeTowerImageProcessingTester(self				)





     @property
     def 	__snake_case    (       self :							str				)->Any:
      return self.image_processor_tester.prepare_image_processor_dict()





     def 	__snake_case    (       self :							Optional[int]				)->Dict:
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	self.image_processing_class(**self.image_processor_dict				)
      self.assertTrue(hasattr(UpperCamelCase     ,  "image_mean"				)				)
      self.assertTrue(hasattr(UpperCamelCase     ,  "image_std"				)				)
      self.assertTrue(hasattr(UpperCamelCase     ,  "do_normalize"				)				)
      self.assertTrue(hasattr(UpperCamelCase     ,  "do_resize"				)				)
      self.assertTrue(hasattr(UpperCamelCase     ,  "size"				)				)
      self.assertTrue(hasattr(UpperCamelCase     ,  "size_divisor"				)				)





     def 	__snake_case    (       self :							Optional[int]				)->int:
      pass





     def 	__snake_case    (       self :							str				)->Union[str, Any]:
      # Initialize image processor
      __SCREAMING_SNAKE_CASE       :    Any       =	self.image_processing_class(**self.image_processor_dict				)
      # create random PIL images
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	prepare_image_inputs(self.image_processor_tester     ,  equal_resolution=UpperCamelCase				)
      for image in image_inputs:
       self.assertIsInstance(UpperCamelCase     ,  Image.Image				)

      # Test not batched input
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	image_processing(image_inputs[0]     ,  return_tensors="pt"				).pixel_values

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Tuple       =	self.image_processor_tester.get_expected_values(UpperCamelCase				)
      self.assertEqual(
          encoded_images.shape     ,  (1, self.image_processor_tester.num_channels, expected_height, expected_width)     ,  )

      # Test batched
      __SCREAMING_SNAKE_CASE       :    Any       =	image_processing(UpperCamelCase     ,  return_tensors="pt"				).pixel_values

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	self.image_processor_tester.get_expected_values(UpperCamelCase     ,  batched=UpperCamelCase				)
      self.assertEqual(
          encoded_images.shape     ,  (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              expected_height,
              expected_width,
          )     ,  )





     def 	__snake_case    (       self :							List[str]				)->Optional[int]:
      # Initialize image processor
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	self.image_processing_class(**self.image_processor_dict				)
      # create random numpy tensors
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	prepare_image_inputs(self.image_processor_tester     ,  equal_resolution=UpperCamelCase     ,  numpify=UpperCamelCase				)
      for image in image_inputs:
       self.assertIsInstance(UpperCamelCase     ,  np.ndarray				)

      # Test not batched input
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	image_processing(image_inputs[0]     ,  return_tensors="pt"				).pixel_values

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Optional[int]       =	self.image_processor_tester.get_expected_values(UpperCamelCase				)
      self.assertEqual(
          encoded_images.shape     ,  (1, self.image_processor_tester.num_channels, expected_height, expected_width)     ,  )

      # Test batched
      __SCREAMING_SNAKE_CASE       :    str       =	image_processing(UpperCamelCase     ,  return_tensors="pt"				).pixel_values

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    List[Any]       =	self.image_processor_tester.get_expected_values(UpperCamelCase     ,  batched=UpperCamelCase				)
      self.assertEqual(
          encoded_images.shape     ,  (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              expected_height,
              expected_width,
          )     ,  )






     def 	__snake_case    (       self :							Optional[Any]				)->Union[str, Any]:
      # Initialize image processor
      __SCREAMING_SNAKE_CASE       :    Dict       =	self.image_processing_class(**self.image_processor_dict				)
      # create random PyTorch tensors
      __SCREAMING_SNAKE_CASE       :    List[str]       =	prepare_image_inputs(self.image_processor_tester     ,  equal_resolution=UpperCamelCase     ,  torchify=UpperCamelCase				)
      for image in image_inputs:
       self.assertIsInstance(UpperCamelCase     ,  torch.Tensor				)

      # Test not batched input
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	image_processing(image_inputs[0]     ,  return_tensors="pt"				).pixel_values

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Dict       =	self.image_processor_tester.get_expected_values(UpperCamelCase				)
      self.assertEqual(
          encoded_images.shape     ,  (1, self.image_processor_tester.num_channels, expected_height, expected_width)     ,  )

      # Test batched
      __SCREAMING_SNAKE_CASE       :    str       =	image_processing(UpperCamelCase     ,  return_tensors="pt"				).pixel_values

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Any       =	self.image_processor_tester.get_expected_values(UpperCamelCase     ,  batched=UpperCamelCase				)
      self.assertEqual(
          encoded_images.shape     ,  (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              expected_height,
              expected_width,
          )     ,  )






