
'''simple docstring'''



import hashlib
import unittest
from typing import Dict

import numpy as np

from transformers import (
    MODEL_FOR_MASK_GENERATION_MAPPING,
    TF_MODEL_FOR_MASK_GENERATION_MAPPING,
    is_vision_available,
    pipeline,
)
from transformers.pipelines import MaskGenerationPipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)


if is_vision_available():
		from PIL import Image
else:





		class _UpperCamelCase		:





						'''simple docstring'''





						@staticmethod
						def 				__lowerCamelCase     (							*_lowerCAmelCase  :  List[Any]  ,       **_lowerCAmelCase  :  str):







													'''simple docstring'''

													pass



def   _A       (   _lowerCAmelCase					):



							"""simple docstring"""



							__lowercase       	=hashlib.mda(image.tobytes()					)
							return m.hexdigest()[:10]



def   _A       (   _lowerCAmelCase					):



							"""simple docstring"""



							__lowercase       	=np.array(_lowerCAmelCase					)
							__lowercase       	=npimg.shape
							return {"hash": hashimage(_lowerCAmelCase					), "shape": shape}





@is_pipeline_test
@require_vision
@require_torch
class _UpperCamelCase		(     unittest.TestCase		):





				'''simple docstring'''





				lowerCAmelCase__					    =    dict(
				    (list(MODEL_FOR_MASK_GENERATION_MAPPING.items()		) if MODEL_FOR_MASK_GENERATION_MAPPING else [])		)
				lowerCAmelCase__					    =    dict(
				    (list(TF_MODEL_FOR_MASK_GENERATION_MAPPING.items()		) if TF_MODEL_FOR_MASK_GENERATION_MAPPING else [])		)

				def 				__lowerCamelCase     (							self  :  Optional[Any]  ,       _lowerCAmelCase  :  List[str]  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Any):







											'''simple docstring'''

											__lowercase       	=MaskGenerationPipeline(model=_lowerCAmelCase  ,       image_processor=_lowerCAmelCase)
											return image_segmenter, [
											    "./tests/fixtures/tests_samples/COCO/000000039769.png",
											    "./tests/fixtures/tests_samples/COCO/000000039769.png",
											]

				def 				__lowerCamelCase     (							self  :  List[str]  ,       _lowerCAmelCase  :  int  ,       _lowerCAmelCase  :  Dict):







											'''simple docstring'''

											pass

				@require_tf
				@unittest.skip('Image segmentation not implemented in TF')
				def 				__lowerCamelCase     (							self  :  List[Any]):







											'''simple docstring'''

											pass

				@slow
				@require_torch
				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=pipeline('mask-generation'  ,       model='facebook/sam-vit-huge')

											__lowercase       	=image_segmenter('http://images.cocodataset.org/val2017/000000039769.jpg'  ,       points_per_batch=2_5_6)

											# Shortening by hashing
											__lowercase       	=[]
											for i, o in enumerate(outputs['masks']):
																		new_outupt += [{"mask": mask_to_test_readable(_lowerCAmelCase), "scores": outputs["scores"][i]}]

											# fmt: off
											self.assertEqual(
											    nested_simplify(_lowerCAmelCase  ,       decimals=4)  ,       [
											        {'mask': {'hash': '115ad19f5f', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0444},
											        {'mask': {'hash': '6affa964c6', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.021},
											        {'mask': {'hash': 'dfe28a0388', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0167},
											        {'mask': {'hash': 'c0a5f4a318', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0132},
											        {'mask': {'hash': 'fe8065c197', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0053},
											        {'mask': {'hash': 'e2d0b7a0b7', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9967},
											        {'mask': {'hash': '453c7844bd', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.993},
											        {'mask': {'hash': '3d44f2926d', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9909},
											        {'mask': {'hash': '64033ddc3f', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9879},
											        {'mask': {'hash': '801064ff79', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9834},
											        {'mask': {'hash': '6172f276ef', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9716},
											        {'mask': {'hash': 'b49e60e084', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9612},
											        {'mask': {'hash': 'a811e775fd', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9599},
											        {'mask': {'hash': 'a6a8ebcf4b', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9552},
											        {'mask': {'hash': '9d8257e080', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9532},
											        {'mask': {'hash': '32de6454a8', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9516},
											        {'mask': {'hash': 'af3d4af2c8', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9499},
											        {'mask': {'hash': '3c6db475fb', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9483},
											        {'mask': {'hash': 'c290813fb9', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9464},
											        {'mask': {'hash': 'b6f0b8f606', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.943},
											        {'mask': {'hash': '92ce16bfdf', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.943},
											        {'mask': {'hash': 'c749b25868', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9408},
											        {'mask': {'hash': 'efb6cab859', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9335},
											        {'mask': {'hash': '1ff2eafb30', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9326},
											        {'mask': {'hash': '788b798e24', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.9262},
											        {'mask': {'hash': 'abea804f0e', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.8999},
											        {'mask': {'hash': '7b9e8ddb73', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.8986},
											        {'mask': {'hash': 'cd24047c8a', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.8984},
											        {'mask': {'hash': '6943e6bcbd', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.8873},
											        {'mask': {'hash': 'b5f47c9191', 'shape': (4_8_0, 6_4_0)}, 'scores': 0.8871}
											    ]  ,       )
											# fmt: on

				@require_torch
				@slow
				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											__lowercase       	='facebook/sam-vit-huge'
											__lowercase       	=pipeline('mask-generation'  ,       model=_lowerCAmelCase)

											__lowercase       	=image_segmenter(
											    'http://images.cocodataset.org/val2017/000000039769.jpg'  ,       pred_iou_thresh=1  ,       points_per_batch=2_5_6)

											# Shortening by hashing
											__lowercase       	=[]
											for i, o in enumerate(outputs['masks']):
																		new_outupt += [{"mask": mask_to_test_readable(_lowerCAmelCase), "scores": outputs["scores"][i]}]

											self.assertEqual(
											    nested_simplify(_lowerCAmelCase  ,       decimals=4)  ,       [
											        {'mask': {'hash': '115ad19f5f', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0444},
											        {'mask': {'hash': '6affa964c6', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0210},
											        {'mask': {'hash': 'dfe28a0388', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0167},
											        {'mask': {'hash': 'c0a5f4a318', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0132},
											        {'mask': {'hash': 'fe8065c197', 'shape': (4_8_0, 6_4_0)}, 'scores': 1.0053},
											    ]  ,       )








'''simple docstring'''



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)

lowerCamelCase	 =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase	 =       {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase	 =       {
    """Salesforce/codegen-350M-mono""": 2048,
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    ["""input_ids""", """attention_mask"""]
				lowerCAmelCase__					    =    CodeGenTokenizer

				def __init__(							self  :  int  ,       _lowerCAmelCase  :  str=None  ,       _lowerCAmelCase  :  List[Any]=None  ,       _lowerCAmelCase  :  Optional[Any]=None  ,       _lowerCAmelCase  :  List[str]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[Any]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]=False  ,       **_lowerCAmelCase  :  Union[str, Any]  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       bos_token=_lowerCAmelCase  ,       eos_token=_lowerCAmelCase  ,       add_prefix_space=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if kwargs.pop('add_bos_token'  ,       _lowerCAmelCase):
																		__lowercase       	=kwargs.pop('name_or_path'  ,       '')
																		raise ValueError(
																		    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
																		    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
																		    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
																		    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
																		    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
																		    ' so that the fast tokenizer works correctly.')

											__lowercase       	=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																		__lowercase       	=getattr(_lowerCAmelCase  ,       pre_tok_state.pop('type'))
																		__lowercase       	=add_prefix_space
																		__lowercase       	=pre_tok_class(**_lowerCAmelCase)

											__lowercase       	=add_prefix_space

				def 				__lowerCamelCase     (							self  :  Any  ,       *_lowerCAmelCase  :  Tuple  ,       **_lowerCAmelCase  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)
											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[str]  ,       *_lowerCAmelCase  :  List[Any]  ,       **_lowerCAmelCase  :  List[str]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]  ,       _lowerCAmelCase  :  bool = False  ,       _lowerCAmelCase  :  bool = None  ,       _lowerCAmelCase  :  Optional[List[str]] = None  ,       **_lowerCAmelCase  :  Any  ,       ):







											'''simple docstring'''

											__lowercase       	=super().decode(
											    token_ids=_lowerCAmelCase  ,       skip_special_tokens=_lowerCAmelCase  ,       clean_up_tokenization_spaces=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if truncate_before_pattern is not None and len(_lowerCAmelCase) > 0:
																		__lowercase       	=self.truncate(_lowerCAmelCase  ,       _lowerCAmelCase)

											return decoded_text

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											def find_re(_lowerCAmelCase  :  List[Any]  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  str):
																		__lowercase       	=pattern.search(_lowerCAmelCase  ,       _lowerCAmelCase)
																		return m.start() if m else -1

											__lowercase       	=[re.compile(_lowerCAmelCase  ,       re.MULTILINE) for pattern in truncate_before_pattern]

											__lowercase       	=list(re.finditer('^print'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: prints[1].start()]

											__lowercase       	=list(re.finditer('^def'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: defs[1].start()]

											__lowercase       	=0

											__lowercase       	=[
											    pos for pos in [find_re(_lowerCAmelCase  ,       _lowerCAmelCase  ,       _lowerCAmelCase) for terminal in terminals] if pos != -1
											]

											if len(_lowerCAmelCase) > 0:
																		return completion[: min(_lowerCAmelCase)]
											else:
																		return completion



