'''simple docstring'''




import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
 from PIL import Image

 from transformers import CLIPSegProcessor, ViTImageProcessor


@require_vision
class       UpperCAmelCase__  (       unittest.TestCase			):






   """simple docstring"""
   def 		_lowerCamelCase (self				)    ->     Dict:
        lowercase_					:       Any     				=					tempfile.mkdtemp()

        # fmt: off
        lowercase_					:       Any     				=					['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
        # fmt: on
        lowercase_					:       List[Any]     				=					dict(zip(_a       ,				range(len(_a				)				)				)				)
        lowercase_					:       List[str]     				=					['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>', '']
        lowercase_					:       Any     				=					{'unk_token': '<unk>'}

        lowercase_					:       Optional[int]     				=					os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['vocab_file']				)
        lowercase_					:       Dict     				=					os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['merges_file']				)
        with open(self.vocab_file       ,				'w'       ,				encoding='utf-8'				) as fp:
             fp.write(json.dumps(_a				) + '\n'				)
        with open(self.merges_file       ,				'w'       ,				encoding='utf-8'				) as fp:
             fp.write('\n'.join(_a				)				)

        lowercase_					:       Any     				=					{
            'do_resize': True,
            'size': 20,
            'do_center_crop': True,
            'crop_size': 18,
            'do_normalize': True,
            'image_mean': [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
            'image_std': [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
        }
        lowercase_					:       str     				=					os.path.join(self.tmpdirname       ,				_a				)
        with open(self.image_processor_file       ,				'w'       ,				encoding='utf-8'				) as fp:
             json.dump(_a       ,				_a				)
   def 		_lowerCamelCase (self       ,				**_a				)    ->     str:
        return CLIPTokenizer.from_pretrained(self.tmpdirname       ,				**_a				)
   def 		_lowerCamelCase (self       ,				**_a				)    ->     Optional[int]:
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,				**_a				)
   def 		_lowerCamelCase (self       ,				**_a				)    ->     str:
        return ViTImageProcessor.from_pretrained(self.tmpdirname       ,				**_a				)
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        shutil.rmtree(self.tmpdirname				)
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        lowercase_					:       Union[str, Any]     				=					[np.random.randint(255       ,				size=(3, 30, 400)       ,				dtype=np.uinta				)]

        lowercase_					:       int     				=					[Image.fromarray(np.moveaxis(_a       ,				0       ,				-1				)				) for x in image_inputs]

        return image_inputs
   def 		_lowerCamelCase (self				)    ->     int:
        lowercase_					:       Union[str, Any]     				=					self.get_tokenizer()
        lowercase_					:       Any     				=					self.get_rust_tokenizer()
        lowercase_					:       int     				=					self.get_image_processor()

        lowercase_					:       str     				=					CLIPSegProcessor(tokenizer=_a       ,				image_processor=_a				)
        processor_slow.save_pretrained(self.tmpdirname				)
        lowercase_					:       str     				=					CLIPSegProcessor.from_pretrained(self.tmpdirname       ,				use_fast=_a				)

        lowercase_					:       List[Any]     				=					CLIPSegProcessor(tokenizer=_a       ,				image_processor=_a				)
        processor_fast.save_pretrained(self.tmpdirname				)
        lowercase_					:       int     				=					CLIPSegProcessor.from_pretrained(self.tmpdirname				)

        self.assertEqual(processor_slow.tokenizer.get_vocab()       ,				tokenizer_slow.get_vocab()				)
        self.assertEqual(processor_fast.tokenizer.get_vocab()       ,				tokenizer_fast.get_vocab()				)
        self.assertEqual(tokenizer_slow.get_vocab()       ,				tokenizer_fast.get_vocab()				)
        self.assertIsInstance(processor_slow.tokenizer       ,				_a				)
        self.assertIsInstance(processor_fast.tokenizer       ,				_a				)

        self.assertEqual(processor_slow.image_processor.to_json_string()       ,				image_processor.to_json_string()				)
        self.assertEqual(processor_fast.image_processor.to_json_string()       ,				image_processor.to_json_string()				)
        self.assertIsInstance(processor_slow.image_processor       ,				_a				)
        self.assertIsInstance(processor_fast.image_processor       ,				_a				)
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        lowercase_					:       str     				=					CLIPSegProcessor(tokenizer=self.get_tokenizer()       ,				image_processor=self.get_image_processor()				)
        processor.save_pretrained(self.tmpdirname				)

        lowercase_					:       List[str]     				=					self.get_tokenizer(bos_token='(BOS)'       ,				eos_token='(EOS)'				)
        lowercase_					:       Dict     				=					self.get_image_processor(do_normalize=_a       ,				padding_value=1.0				)

        lowercase_					:       Dict     				=					CLIPSegProcessor.from_pretrained(
            self.tmpdirname       ,				bos_token='(BOS)'       ,				eos_token='(EOS)'       ,				do_normalize=_a       ,				padding_value=1.0				)

        self.assertEqual(processor.tokenizer.get_vocab()       ,				tokenizer_add_kwargs.get_vocab()				)
        self.assertIsInstance(processor.tokenizer       ,				_a				)

        self.assertEqual(processor.image_processor.to_json_string()       ,				image_processor_add_kwargs.to_json_string()				)
        self.assertIsInstance(processor.image_processor       ,				_a				)
   def 		_lowerCamelCase (self				)    ->     Union[str, Any]:
        lowercase_					:       Dict     				=					self.get_image_processor()
        lowercase_					:       List[Any]     				=					self.get_tokenizer()

        lowercase_					:       int     				=					CLIPSegProcessor(tokenizer=_a       ,				image_processor=_a				)

        lowercase_					:       Any     				=					self.prepare_image_inputs()

        lowercase_					:       str     				=					image_processor(_a       ,				return_tensors='np'				)
        lowercase_					:       List[Any]     				=					processor(images=_a       ,				return_tensors='np'				)

        for key in input_feat_extract.keys():
             self.assertAlmostEqual(input_feat_extract[key].sum()       ,				input_processor[key].sum()       ,				delta=1e-2				)
   def 		_lowerCamelCase (self				)    ->     Dict:
        lowercase_					:       Dict     				=					self.get_image_processor()
        lowercase_					:       Tuple     				=					self.get_tokenizer()

        lowercase_					:       Dict     				=					CLIPSegProcessor(tokenizer=_a       ,				image_processor=_a				)

        lowercase_					:       Dict     				=					'lower newer'

        lowercase_					:       Union[str, Any]     				=					processor(text=_a				)

        lowercase_					:       Tuple     				=					tokenizer(_a				)

        for key in encoded_tok.keys():
             self.assertListEqual(encoded_tok[key]       ,				encoded_processor[key]				)
   def 		_lowerCamelCase (self				)    ->     Union[str, Any]:
        lowercase_					:       int     				=					self.get_image_processor()
        lowercase_					:       Dict     				=					self.get_tokenizer()

        lowercase_					:       str     				=					CLIPSegProcessor(tokenizer=_a       ,				image_processor=_a				)

        lowercase_					:       Any     				=					'lower newer'
        lowercase_					:       Optional[int]     				=					self.prepare_image_inputs()

        lowercase_					:       int     				=					processor(text=_a       ,				images=_a				)

        self.assertListEqual(list(inputs.keys()				)       ,				['input_ids', 'attention_mask', 'pixel_values']				)

        # test if it raises when no input is passed
        with pytest.raises(_a				):
             processor()
   def 		_lowerCamelCase (self				)    ->     Optional[Any]:
        lowercase_					:       Union[str, Any]     				=					self.get_image_processor()
        lowercase_					:       str     				=					self.get_tokenizer()

        lowercase_					:       Tuple     				=					CLIPSegProcessor(tokenizer=_a       ,				image_processor=_a				)

        lowercase_					:       List[Any]     				=					self.prepare_image_inputs()
        lowercase_					:       Union[str, Any]     				=					self.prepare_image_inputs()

        lowercase_					:       int     				=					processor(images=_a       ,				visual_prompt=_a				)

        self.assertListEqual(list(inputs.keys()				)       ,				['pixel_values', 'conditional_pixel_values']				)

        # test if it raises when no input is passed
        with pytest.raises(_a				):
             processor()





   def 		_lowerCamelCase (self				)    ->     Dict:
        lowercase_					:       Optional[Any]     				=					self.get_image_processor()
        lowercase_					:       Tuple     				=					self.get_tokenizer()

        lowercase_					:       str     				=					CLIPSegProcessor(tokenizer=_a       ,				image_processor=_a				)

        lowercase_					:       Union[str, Any]     				=					[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        lowercase_					:       int     				=					processor.batch_decode(_a				)
        lowercase_					:       Union[str, Any]     				=					tokenizer.batch_decode(_a				)

        self.assertListEqual(_a       ,				_a				)





'''simple docstring'''




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
 import botoa  # noqa: F401

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):
     lowercase_					:       str     				=					botoa.client('iam'							)

     lowercase_					:       Optional[int]     				=					{
         'Version': '2012-10-17',
         'Statement': [
             {'Effect': 'Allow', 'Principal': {'Service': 'sagemaker.amazonaws.com'}, 'Action': 'sts:AssumeRole'}
         ],
     }
     try:
          # create the role, associated with the chosen trust policy
          iam_client.create_role(
              RoleName=SCREAMING_SNAKE_CASE_  ,       AssumeRolePolicyDocument=json.dumps(SCREAMING_SNAKE_CASE_  ,       indent=2							)							)
          lowercase_					:       Optional[Any]     				=					{
              'Version': '2012-10-17',
              'Statement': [
                  {
                      'Effect': 'Allow',
                      'Action': [
                          'sagemaker:*',
                          'ecr:GetDownloadUrlForLayer',
                          'ecr:BatchGetImage',
                          'ecr:BatchCheckLayerAvailability',
                          'ecr:GetAuthorizationToken',
                          'cloudwatch:PutMetricData',
                          'cloudwatch:GetMetricData',
                          'cloudwatch:GetMetricStatistics',
                          'cloudwatch:ListMetrics',
                          'logs:CreateLogGroup',
                          'logs:CreateLogStream',
                          'logs:DescribeLogStreams',
                          'logs:PutLogEvents',
                          'logs:GetLogEvents',
                          's3:CreateBucket',
                          's3:ListBucket',
                          's3:GetBucketLocation',
                          's3:GetObject',
                          's3:PutObject',
                      ],
                      'Resource': '*',
                  }
              ],
          }
          # attach policy to role
          iam_client.put_role_policy(
              RoleName=SCREAMING_SNAKE_CASE_  ,       PolicyName=f'''{role_name}_policy_permission'''  ,       PolicyDocument=json.dumps(SCREAMING_SNAKE_CASE_  ,       indent=2							)  ,       )
     except iam_client.exceptions.EntityAlreadyExistsException:
          print(f'''role {role_name} already exists. Using existing one'''							)

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):
     lowercase_					:       List[Any]     				=					botoa.client('iam'							)
     return iam_client.get_role(RoleName=SCREAMING_SNAKE_CASE_							)["Role"]["Arn"]






def _UpperCamelCase   (       ):
     lowercase_					:       Dict     				=					_ask_options(
         'How do you want to authorize?'  ,       ['AWS Profile', 'Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) ']  ,       SCREAMING_SNAKE_CASE_  ,       )
     lowercase_					:       List[Any]     				=					None
     if credentials_configuration == 0:
          lowercase_					:       List[Any]     				=					_ask_field('Enter your AWS Profile name: [default] '  ,       default='default'							)
          lowercase_					:       Optional[int]     				=					aws_profile
     else:
          print(
              'Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,'
              '`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`'							)
          lowercase_					:       Optional[Any]     				=					_ask_field('AWS Access Key ID: '							)
          lowercase_					:       Optional[int]     				=					aws_access_key_id

          lowercase_					:       Optional[Any]     				=					_ask_field('AWS Secret Access Key: '							)
          lowercase_					:       Optional[int]     				=					aws_secret_access_key

     lowercase_					:       Any     				=					_ask_field('Enter your AWS Region: [us-east-1]'  ,       default='us-east-1'							)
     lowercase_					:       Tuple     				=					aws_region

     lowercase_					:       int     				=					_ask_options(
         'Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?'  ,       ['Provide IAM Role name', 'Create new IAM role using credentials']  ,       SCREAMING_SNAKE_CASE_  ,       )
     if role_management == 0:
          lowercase_					:       int     				=					_ask_field('Enter your IAM role name: '							)
     else:
          lowercase_					:       Any     				=					'accelerate_sagemaker_execution_role'
          print(f'''Accelerate will create an iam role "{iam_role_name}" using the provided credentials'''							)
          _create_iam_role_for_sagemaker(SCREAMING_SNAKE_CASE_							)

     lowercase_					:       List[Any]     				=					_ask_field(
         'Do you want to use custom Docker image? [yes/NO]: '  ,       _convert_yes_no_to_bool  ,       default=SCREAMING_SNAKE_CASE_  ,       error_message='Please enter yes or no.'  ,       )
     lowercase_					:       Optional[Any]     				=					None
     if is_custom_docker_image:
          lowercase_					:       List[Any]     				=					_ask_field('Enter your Docker image: '  ,       lambda SCREAMING_SNAKE_CASE_							: str(SCREAMING_SNAKE_CASE_							).lower()							)

     lowercase_					:       Any     				=					_ask_field(
         'Do you want to provide SageMaker input channels with data locations? [yes/NO]: '  ,       _convert_yes_no_to_bool  ,       default=SCREAMING_SNAKE_CASE_  ,       error_message='Please enter yes or no.'  ,       )
     lowercase_					:       Optional[Any]     				=					None
     if is_sagemaker_inputs_enabled:
          lowercase_					:       Dict     				=					_ask_field(
              'Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): '  ,       lambda SCREAMING_SNAKE_CASE_							: str(SCREAMING_SNAKE_CASE_							).lower()  ,       )

     lowercase_					:       Dict     				=					_ask_field(
         'Do you want to enable SageMaker metrics? [yes/NO]: '  ,       _convert_yes_no_to_bool  ,       default=SCREAMING_SNAKE_CASE_  ,       error_message='Please enter yes or no.'  ,       )
     lowercase_					:       Optional[Any]     				=					None
     if is_sagemaker_metrics_enabled:
          lowercase_					:       Dict     				=					_ask_field(
              'Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): '  ,       lambda SCREAMING_SNAKE_CASE_							: str(SCREAMING_SNAKE_CASE_							).lower()  ,       )

     lowercase_					:       int     				=					_ask_options(
         'What is the distributed mode?'  ,       ['No distributed training', 'Data parallelism']  ,       _convert_sagemaker_distributed_mode  ,       )
     lowercase_					:       List[str]     				=					{}
     lowercase_					:       Optional[Any]     				=					_ask_field(
         'Do you wish to optimize your script with torch dynamo?[yes/NO]:'  ,       _convert_yes_no_to_bool  ,       default=SCREAMING_SNAKE_CASE_  ,       error_message='Please enter yes or no.'  ,       )
     if use_dynamo:
          lowercase_					:       List[Any]     				=					'dynamo_'
          lowercase_					:       List[Any]     				=					_ask_options(
              'Which dynamo backend would you like to use?'  ,       [x.lower() for x in DYNAMO_BACKENDS]  ,       _convert_dynamo_backend  ,       default=2  ,       )
          lowercase_					:       int     				=					_ask_field(
              'Do you want to customize the defaults sent to torch.compile? [yes/NO]: '  ,       _convert_yes_no_to_bool  ,       default=SCREAMING_SNAKE_CASE_  ,       error_message='Please enter yes or no.'  ,       )

          if use_custom_options:
               lowercase_					:       Union[str, Any]     				=					_ask_options(
                   'Which mode do you want to use?'  ,       SCREAMING_SNAKE_CASE_  ,       lambda SCREAMING_SNAKE_CASE_							: TORCH_DYNAMO_MODES[int(SCREAMING_SNAKE_CASE_							)]  ,       default='default'  ,       )
               lowercase_					:       Union[str, Any]     				=					_ask_field(
                   'Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: '  ,       _convert_yes_no_to_bool  ,       default=SCREAMING_SNAKE_CASE_  ,       error_message='Please enter yes or no.'  ,       )
               lowercase_					:       Tuple     				=					_ask_field(
                   'Do you want to enable dynamic shape tracing? [yes/NO]: '  ,       _convert_yes_no_to_bool  ,       default=SCREAMING_SNAKE_CASE_  ,       error_message='Please enter yes or no.'  ,       )
     lowercase_					:       List[str]     				=					'Which EC2 instance type you want to use for your training?'
     if distributed_type != SageMakerDistributedType.NO:
          lowercase_					:       Optional[Any]     				=					_ask_options(
              SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_  ,       lambda SCREAMING_SNAKE_CASE_							: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(SCREAMING_SNAKE_CASE_							)]							)
     else:
          eca_instance_query += "? [ml.p3.2xlarge]:"
          lowercase_					:       Tuple     				=					_ask_field(SCREAMING_SNAKE_CASE_  ,       lambda SCREAMING_SNAKE_CASE_							: str(SCREAMING_SNAKE_CASE_							).lower()  ,       default='ml.p3.2xlarge'							)

     lowercase_					:       str     				=					1
     if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
          lowercase_					:       Union[str, Any]     				=					_ask_field(
              'How many machines do you want use? [1]: '  ,       SCREAMING_SNAKE_CASE_  ,       default=1  ,       )

     lowercase_					:       Any     				=					_ask_options(
         'Do you wish to use FP16 or BF16 (mixed precision)?'  ,       ['no', 'fp16', 'bf16', 'fp8']  ,       _convert_mixed_precision  ,       )

     if use_dynamo and mixed_precision == "no":
          print(
              'Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts.'							)

     return SageMakerConfig(
         image_uri=SCREAMING_SNAKE_CASE_  ,       compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER  ,       distributed_type=SCREAMING_SNAKE_CASE_  ,       use_cpu=SCREAMING_SNAKE_CASE_  ,       dynamo_config=SCREAMING_SNAKE_CASE_  ,       eca_instance_type=SCREAMING_SNAKE_CASE_  ,       profile=SCREAMING_SNAKE_CASE_  ,       region=SCREAMING_SNAKE_CASE_  ,       iam_role_name=SCREAMING_SNAKE_CASE_  ,       mixed_precision=SCREAMING_SNAKE_CASE_  ,       num_machines=SCREAMING_SNAKE_CASE_  ,       sagemaker_inputs_file=SCREAMING_SNAKE_CASE_  ,       sagemaker_metrics_file=SCREAMING_SNAKE_CASE_  ,       )




