





"""simple docstring"""






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast



class 	lowerCAmelCase					(      unittest.TestCase							):






       def 							__A (   self     ):
              _UpperCAmelCase     =     0






       @slow
       def 							__A (   self     ):
              for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
                     self.assertIsNotNone(a__     )
                     self.assertIsInstance(a__    ,  (BertTokenizer, BertTokenizerFast)     )
                     self.assertGreater(len(a__     )    ,  0     )

              for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
                     self.assertIsNotNone(a__     )
                     self.assertIsInstance(a__    ,  (GPTaTokenizer, GPTaTokenizerFast)     )
                     self.assertGreater(len(a__     )    ,  0     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
              self.assertIsInstance(a__    ,  (BertTokenizer, BertTokenizerFast)     )
              self.assertEqual(tokenizer.vocab_size    ,  12     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
              self.assertIsInstance(a__    ,  (RobertaTokenizer, RobertaTokenizerFast)     )
              self.assertEqual(tokenizer.vocab_size    ,  20     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoConfig.from_pretrained(a__     )
              self.assertIsInstance(a__    ,  a__     )
              # Check that tokenizer_type ≠ model_type
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  config=a__     )
              self.assertIsInstance(a__    ,  (BertTokenizer, BertTokenizerFast)     )
              self.assertEqual(tokenizer.vocab_size    ,  12     )






       def 							__A (   self     ):
              with tempfile.TemporaryDirectory() as tmp_dir:
                     shutil.copy('./tests/fixtures/vocab.txt'    ,  os.path.join(a__    ,  'vocab.txt'     )     )

                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  tokenizer_type='bert'    ,  use_fast=a__     )
                     self.assertIsInstance(a__    ,  a__     )

              with tempfile.TemporaryDirectory() as tmp_dir:
                     shutil.copy('./tests/fixtures/vocab.json'    ,  os.path.join(a__    ,  'vocab.json'     )     )
                     shutil.copy('./tests/fixtures/merges.txt'    ,  os.path.join(a__    ,  'merges.txt'     )     )

                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  tokenizer_type='gpt2'    ,  use_fast=a__     )
                     self.assertIsInstance(a__    ,  a__     )






       @require_tokenizers
       def 							__A (   self     ):
              with tempfile.TemporaryDirectory() as tmp_dir:
                     shutil.copy('./tests/fixtures/vocab.txt'    ,  os.path.join(a__    ,  'vocab.txt'     )     )

                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  tokenizer_type='bert'     )
                     self.assertIsInstance(a__    ,  a__     )

              with tempfile.TemporaryDirectory() as tmp_dir:
                     shutil.copy('./tests/fixtures/vocab.json'    ,  os.path.join(a__    ,  'vocab.json'     )     )
                     shutil.copy('./tests/fixtures/merges.txt'    ,  os.path.join(a__    ,  'merges.txt'     )     )

                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  tokenizer_type='gpt2'     )
                     self.assertIsInstance(a__    ,  a__     )






       def 							__A (   self     ):
              with pytest.raises(a__     ):
                     AutoTokenizer.from_pretrained('./'    ,  tokenizer_type='xxx'     )






       @require_tokenizers
       def 							__A (   self     ):
              for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                     _UpperCAmelCase     =     tokenizer_class.from_pretrained('wietsedv/bert-base-dutch-cased'     )
                     self.assertIsInstance(a__    ,  (BertTokenizer, BertTokenizerFast)     )

                     if isinstance(a__    ,  a__     ):
                            self.assertEqual(tokenizer.basic_tokenizer.do_lower_case    ,  a__     )
                     else:
                            self.assertEqual(tokenizer.do_lower_case    ,  a__     )

                     self.assertEqual(tokenizer.model_max_length    ,  5_12     )






       @require_tokenizers
       def 							__A (   self     ):
              for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                     with self.assertRaisesRegex(
                         a__    ,  'julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'    ,  ):
                            _UpperCAmelCase     =     tokenizer_class.from_pretrained('julien-c/herlolip-not-exists'     )






       def 							__A (   self     ):
              # tests: https://github.com/huggingface/transformers/pull/13251
              # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
              # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
              _UpperCAmelCase     =     TOKENIZER_MAPPING.values()
              _UpperCAmelCase     =     []

              for slow_tok, fast_tok in tokenizers:
                     if slow_tok is not None:
                            tokenizer_names.append(slow_tok.__name__     )

                     if fast_tok is not None:
                            tokenizer_names.append(fast_tok.__name__     )

              for tokenizer_name in tokenizer_names:
                     # must find the right class
                     tokenizer_class_from_name(a__     )






       @require_tokenizers
       def 							__A (   self     ):
              self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'    ,  use_fast=a__     )    ,  a__     )
              self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'     )    ,  a__     )






       @require_tokenizers
       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('distilbert-base-uncased'    ,  do_lower_case=a__     )
              _UpperCAmelCase     =     'Hello, world. How are you?'
              _UpperCAmelCase     =     tokenizer.tokenize(a__     )
              self.assertEqual('[UNK]'    ,  tokens[0]     )

              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('microsoft/mpnet-base'    ,  do_lower_case=a__     )
              _UpperCAmelCase     =     tokenizer.tokenize(a__     )
              self.assertEqual('[UNK]'    ,  tokens[0]     )






       @require_tokenizers
       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('robot-test/dummy-tokenizer-fast-with-model-config'     )
              self.assertEqual(type(a__     )    ,  a__     )
              self.assertEqual(tokenizer.model_max_length    ,  5_12     )
              self.assertEqual(tokenizer.vocab_size    ,  3_00_00     )
              self.assertEqual(tokenizer.unk_token    ,  '[UNK]'     )
              self.assertEqual(tokenizer.padding_side    ,  'right'     )
              self.assertEqual(tokenizer.truncation_side    ,  'right'     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
              self.assertIsInstance(a__    ,  (BertTokenizer, BertTokenizerFast)     )
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(a__     )
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )

              self.assertIsInstance(a__    ,  tokenizer.__class__     )
              self.assertEqual(tokenizera.vocab_size    ,  12     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('ctrl'     )
              # There is no fast CTRL so this always gives us a slow tokenizer.
              self.assertIsInstance(a__    ,  a__     )






       def 							__A (   self     ):
              # Check we can load the tokenizer config of an online model.
              _UpperCAmelCase     =     get_tokenizer_config('bert-base-cased'     )
              _UpperCAmelCase     =     config.pop('_commit_hash'    ,  a__     )
              # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
              self.assertEqual(a__    ,  {'do_lower_case': False}     )

              # This model does not have a tokenizer_config so we get back an empty dict.
              _UpperCAmelCase     =     get_tokenizer_config(a__     )
              self.assertDictEqual(a__    ,  {}     )

              # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(a__     )
                     _UpperCAmelCase     =     get_tokenizer_config(a__     )

              # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
              self.assertEqual(config['tokenizer_class']    ,  'BertTokenizer'     )






       def 							__A (   self     ):
              try:
                     AutoConfig.register('custom'    ,  a__     )

                     AutoTokenizer.register(a__    ,  slow_tokenizer_class=a__     )
                     # Trying to register something existing in the Transformers library will raise an error
                     with self.assertRaises(a__     ):
                            AutoTokenizer.register(a__    ,  slow_tokenizer_class=a__     )

                     _UpperCAmelCase     =     CustomTokenizer.from_pretrained(a__     )
                     with tempfile.TemporaryDirectory() as tmp_dir:
                            tokenizer.save_pretrained(a__     )

                            _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
                            self.assertIsInstance(a__    ,  a__     )

              finally:
                     if "custom" in CONFIG_MAPPING._extra_content:
                            del CONFIG_MAPPING._extra_content["custom"]
                     if CustomConfig in TOKENIZER_MAPPING._extra_content:
                            del TOKENIZER_MAPPING._extra_content[CustomConfig]






       @require_tokenizers
       def 							__A (   self     ):
              try:
                     AutoConfig.register('custom'    ,  a__     )

                     # Can register in two steps
                     AutoTokenizer.register(a__    ,  slow_tokenizer_class=a__     )
                     self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,  (CustomTokenizer, None)     )
                     AutoTokenizer.register(a__    ,  fast_tokenizer_class=a__     )
                     self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,  (CustomTokenizer, CustomTokenizerFast)     )

                     del TOKENIZER_MAPPING._extra_content[CustomConfig]
                     # Can register in one step
                     AutoTokenizer.register(
                         a__    ,  slow_tokenizer_class=a__    ,  fast_tokenizer_class=a__     )
                     self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,  (CustomTokenizer, CustomTokenizerFast)     )

                     # Trying to register something existing in the Transformers library will raise an error
                     with self.assertRaises(a__     ):
                            AutoTokenizer.register(a__    ,  fast_tokenizer_class=a__     )

                     # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
                     # and that model does not have a tokenizer.json
                     with tempfile.TemporaryDirectory() as tmp_dir:
                            _UpperCAmelCase     =     BertTokenizerFast.from_pretrained(a__     )
                            bert_tokenizer.save_pretrained(a__     )
                            _UpperCAmelCase     =     CustomTokenizerFast.from_pretrained(a__     )

                     with tempfile.TemporaryDirectory() as tmp_dir:
                            tokenizer.save_pretrained(a__     )

                            _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__     )
                            self.assertIsInstance(a__    ,  a__     )

                            _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  use_fast=a__     )
                            self.assertIsInstance(a__    ,  a__     )

              finally:
                     if "custom" in CONFIG_MAPPING._extra_content:
                            del CONFIG_MAPPING._extra_content["custom"]
                     if CustomConfig in TOKENIZER_MAPPING._extra_content:
                            del TOKENIZER_MAPPING._extra_content[CustomConfig]






       def 							__A (   self     ):
              # If remote code is not set, we will time out when asking whether to load the model.
              with self.assertRaises(a__     ):
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'     )
              # If remote code is disabled, we can't load this config.
              with self.assertRaises(a__     ):
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                         'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=a__     )

              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=a__     )
              self.assertTrue(tokenizer.special_attribute_present     )
              # Test tokenizer can be reloaded.
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(a__     )
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  trust_remote_code=a__     )
              self.assertTrue(reloaded_tokenizer.special_attribute_present     )

              if is_tokenizers_available():
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'     )
                     self.assertEqual(reloaded_tokenizer.__class__.__name__    ,  'NewTokenizerFast'     )

                     # Test we can also load the slow version
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                         'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=a__    ,  use_fast=a__     )
                     self.assertTrue(tokenizer.special_attribute_present     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'     )
                     # Test tokenizer can be reloaded.
                     with tempfile.TemporaryDirectory() as tmp_dir:
                            tokenizer.save_pretrained(a__     )
                            _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  trust_remote_code=a__    ,  use_fast=a__     )
                     self.assertEqual(reloaded_tokenizer.__class__.__name__    ,  'NewTokenizer'     )
                     self.assertTrue(reloaded_tokenizer.special_attribute_present     )
              else:
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'     )
                     self.assertEqual(reloaded_tokenizer.__class__.__name__    ,  'NewTokenizer'     )






       @require_tokenizers
       def 							__A (   self     ):



              class 	lowerCAmelCase					(      snake_case							):
                     lowerCAmelCase__						 =  False



              class 	lowerCAmelCase					(      snake_case							):
                     lowerCAmelCase__						 =  NewTokenizer
                     lowerCAmelCase__						 =  False

              try:
                     AutoConfig.register('custom'    ,  a__     )
                     AutoTokenizer.register(a__    ,  slow_tokenizer_class=a__     )
                     AutoTokenizer.register(a__    ,  fast_tokenizer_class=a__     )
                     # If remote code is not set, the default is to use local
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'     )
                     self.assertFalse(tokenizer.special_attribute_present     )
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    ,  use_fast=a__     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'     )
                     self.assertFalse(tokenizer.special_attribute_present     )

                     # If remote code is disabled, we load the local one.
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                         'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=a__     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'     )
                     self.assertFalse(tokenizer.special_attribute_present     )
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                         'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=a__    ,  use_fast=a__     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'     )
                     self.assertFalse(tokenizer.special_attribute_present     )

                     # If remote is enabled, we load from the Hub
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                         'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=a__     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'     )
                     self.assertTrue(tokenizer.special_attribute_present     )
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                         'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=a__    ,  use_fast=a__     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'     )
                     self.assertTrue(tokenizer.special_attribute_present     )

              finally:
                     if "custom" in CONFIG_MAPPING._extra_content:
                            del CONFIG_MAPPING._extra_content["custom"]
                     if CustomConfig in TOKENIZER_MAPPING._extra_content:
                            del TOKENIZER_MAPPING._extra_content[CustomConfig]






       def 							__A (   self     ):
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                  'hf-internal-testing/test_dynamic_tokenizer_legacy'    ,  trust_remote_code=a__     )
              self.assertTrue(tokenizer.special_attribute_present     )
              if is_tokenizers_available():
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'     )

                     # Test we can also load the slow version
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(
                         'hf-internal-testing/test_dynamic_tokenizer_legacy'    ,  trust_remote_code=a__    ,  use_fast=a__     )
                     self.assertTrue(tokenizer.special_attribute_present     )
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'     )
              else:
                     self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'     )






       def 							__A (   self     ):
              with self.assertRaisesRegex(
                  a__    ,  'bert-base is not a local folder and is not a valid model identifier'     ):
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained('bert-base'     )






       def 							__A (   self     ):
              with self.assertRaisesRegex(
                  a__    ,  R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'     ):
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained(a__    ,  revision='aaaaaa'     )






       def 							__A (   self     ):
              # Make sure we have cached the tokenizer.
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'     )
              with RequestCounter() as counter:
                     _UpperCAmelCase     =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'     )
                     self.assertEqual(counter.get_request_count    ,  0     )
                     self.assertEqual(counter.head_request_count    ,  1     )
                     self.assertEqual(counter.other_request_count    ,  0     )













"""simple docstring"""






# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import torch

from ..models.clipseg import CLIPSegForImageSegmentation
from ..utils import is_vision_available, requires_backends
from .base import PipelineTool


if is_vision_available():
    from PIL import Image



class 	lowerCAmelCase					(      snake_case							):
       lowerCAmelCase__						 =  (
           """This is a tool that creates a segmentation mask of an image according to a label. It cannot create an image."""
           """It takes two arguments named `image` which should be the original image, and `label` which should be a text """
           """describing the elements what should be identified in the segmentation mask. The tool returns the mask."""
       )
       lowerCAmelCase__						 =  """CIDAS/clipseg-rd64-refined"""
       lowerCAmelCase__						 =  """image_segmenter"""
       lowerCAmelCase__						 =  CLIPSegForImageSegmentation

       lowerCAmelCase__						 =  ["""image""", """text"""]
       lowerCAmelCase__						 =  ["""image"""]






       def __init__(   self    ,  *a__    ,  **a__     ):
              requires_backends(self    ,  ['vision']     )
              super().__init__(*a__    ,  **a__     )






       def 							__A (   self    ,  a__    ,  a__     ):
              return self.pre_processor(text=[label]    ,  images=[image]    ,  padding=a__    ,  return_tensors='pt'     )






       def 							__A (   self    ,  a__     ):
              with torch.no_grad():
                     _UpperCAmelCase     =     self.model(**a__     ).logits
              return logits






       def 							__A (   self    ,  a__     ):
              _UpperCAmelCase     =     outputs.cpu().detach().numpy()
              _UpperCAmelCase     =     0
              _UpperCAmelCase     =     1
              return Image.fromarray((array * 2_55).astype(np.uinta     )     )






