



"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_big_bird import BigBirdTokenizer
else:
					__lowerCAmelCase       :					Optional[int]      					=					None

__lowerCAmelCase       :					Union[str, Any]      					=					logging.get_logger(__name__)
__lowerCAmelCase       :					str      					=					{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCAmelCase       :					Optional[Any]      					=					{
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    },
    '''tokenizer_file''': {
        '''google/bigbird-roberta-base''': (
            '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json'''
        ),
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json'''
        ),
    },
}

__lowerCAmelCase       :					Any      					=					{
    '''google/bigbird-roberta-base''': 4096,
    '''google/bigbird-roberta-large''': 4096,
    '''google/bigbird-base-trivia-itc''': 4096,
}


__lowerCAmelCase       :					List[Any]      					=					'''▁'''



class    _lowerCAmelCase    (				SCREAMING_SNAKE_CASE__		):





		"""simple docstring"""






		_lowerCamelCase			  =       VOCAB_FILES_NAMES
		_lowerCamelCase			  =       PRETRAINED_VOCAB_FILES_MAP
		_lowerCamelCase			  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_lowerCamelCase			  =       BigBirdTokenizer
		_lowerCamelCase			  =       ['''input_ids''', '''attention_mask''']
		_lowerCamelCase			  =       []





		def __init__(					self							,    _lowercase=None							,    _lowercase=None							,    _lowercase="<unk>"							,    _lowercase="<s>"							,    _lowercase="</s>"							,    _lowercase="<pad>"							,    _lowercase="[SEP]"							,    _lowercase="[MASK]"							,    _lowercase="[CLS]"							,    **_lowercase							,    ) ->						int:







								'''simple docstring'''

								snake_case_   : Tuple						=			AddedToken(_lowercase							,    lstrip=_lowercase							,    rstrip=_lowercase   ) if isinstance(_lowercase							,    _lowercase   ) else bos_token
								snake_case_   : Dict						=			AddedToken(_lowercase							,    lstrip=_lowercase							,    rstrip=_lowercase   ) if isinstance(_lowercase							,    _lowercase   ) else eos_token
								snake_case_   : List[Any]						=			AddedToken(_lowercase							,    lstrip=_lowercase							,    rstrip=_lowercase   ) if isinstance(_lowercase							,    _lowercase   ) else unk_token
								snake_case_   : Union[str, Any]						=			AddedToken(_lowercase							,    lstrip=_lowercase							,    rstrip=_lowercase   ) if isinstance(_lowercase							,    _lowercase   ) else pad_token
								snake_case_   : Tuple						=			AddedToken(_lowercase							,    lstrip=_lowercase							,    rstrip=_lowercase   ) if isinstance(_lowercase							,    _lowercase   ) else cls_token
								snake_case_   : List[Any]						=			AddedToken(_lowercase							,    lstrip=_lowercase							,    rstrip=_lowercase   ) if isinstance(_lowercase							,    _lowercase   ) else sep_token

								# Mask token behave like a normal word, i.e. include the space before it
								snake_case_   : Optional[int]						=			AddedToken(_lowercase							,    lstrip=_lowercase							,    rstrip=_lowercase   ) if isinstance(_lowercase							,    _lowercase   ) else mask_token

								super().__init__(
								    _lowercase							,    tokenizer_file=_lowercase							,    bos_token=_lowercase							,    eos_token=_lowercase							,    unk_token=_lowercase							,    sep_token=_lowercase							,    pad_token=_lowercase							,    cls_token=_lowercase							,    mask_token=_lowercase							,    **_lowercase							,    )

								snake_case_   : Optional[Any]						=			vocab_file
								snake_case_   : Optional[int]						=			False if not self.vocab_file else True





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = None   ) ->						List[int]:







								'''simple docstring'''

								snake_case_   : List[str]						=			[self.sep_token_id]
								snake_case_   : Dict						=			[self.cls_token_id]
								if token_ids_a is None:
														return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = None							,    _lowercase = False   ) ->						List[int]:







								'''simple docstring'''

								if already_has_special_tokens:
														if token_ids_a is not None:
																				raise ValueError(
																				    """You should not supply a second sequence if the provided sequence of """
																				    """ids is already formatted with special tokens for the model."""   )
														return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is None:
														return [1] + ([0] * len(_lowercase   )) + [1]
								return [1] + ([0] * len(_lowercase   )) + [1] + ([0] * len(_lowercase   )) + [1]





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = None   ) ->						List[int]:







								'''simple docstring'''

								snake_case_   : List[str]						=			[self.sep_token_id]
								snake_case_   : Optional[Any]						=			[self.cls_token_id]

								if token_ids_a is None:
														return len(cls + token_ids_a + sep   ) * [0]
								return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = None   ) ->						Tuple[str]:







								'''simple docstring'''

								if not self.can_save_slow_tokenizer:
														raise ValueError(
														    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
														    """tokenizer."""   )

								if not os.path.isdir(_lowercase   ):
														logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
														return
								snake_case_   : Dict						=			os.path.join(
								    _lowercase							,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

								if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowercase   ):
														copyfile(self.vocab_file							,    _lowercase   )

								return (out_vocab_file,)






"""simple docstring"""





import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel

from ...utils import logging


__lowerCAmelCase       :					Optional[int]      					=					logging.get_logger(__name__)
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Tuple					, __UpperCamelCase      :			Optional[Any]    ):


						'''simple docstring'''






						snake_case_   : Union[str, Any]						=			nn.functional.normalize(__UpperCamelCase    )
						snake_case_   : Tuple						=			nn.functional.normalize(__UpperCamelCase    )
						return torch.mm(__UpperCamelCase					, normalized_text_embeds.t()    )






class    _lowerCAmelCase    (				SCREAMING_SNAKE_CASE__		):





		"""simple docstring"""






		_lowerCamelCase			  =       CLIPConfig

		_lowerCamelCase			  =       ['''CLIPEncoderLayer''']





		def __init__(					self							,    _lowercase   ) ->						Union[str, Any]:







								'''simple docstring'''

								super().__init__(_lowercase   )

								snake_case_   : Tuple						=			CLIPVisionModel(config.vision_config   )
								snake_case_   : int						=			nn.Linear(config.vision_config.hidden_size							,    config.projection_dim							,    bias=_lowercase   )

								snake_case_   : Optional[Any]						=			nn.Parameter(torch.ones(1_7							,    config.projection_dim   )							,    requires_grad=_lowercase   )
								snake_case_   : Dict						=			nn.Parameter(torch.ones(3							,    config.projection_dim   )							,    requires_grad=_lowercase   )

								snake_case_   : Any						=			nn.Parameter(torch.ones(1_7   )							,    requires_grad=_lowercase   )
								snake_case_   : List[str]						=			nn.Parameter(torch.ones(3   )							,    requires_grad=_lowercase   )





		@torch.no_grad()
		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase   ) ->						Any:







								'''simple docstring'''

								snake_case_   : int						=			self.vision_model(_lowercase   )[1]  # pooled_output
								snake_case_   : str						=			self.visual_projection(_lowercase   )

								# we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
								snake_case_   : Dict						=			cosine_distance(_lowercase							,    self.special_care_embeds   ).cpu().float().numpy()
								snake_case_   : List[str]						=			cosine_distance(_lowercase							,    self.concept_embeds   ).cpu().float().numpy()

								snake_case_   : Any						=			[]
								snake_case_   : Any						=			image_embeds.shape[0]
								for i in range(_lowercase   ):
														snake_case_   : List[Any]						=			{"""special_scores""": {}, """special_care""": [], """concept_scores""": {}, """bad_concepts""": []}

														# increase this value to create a stronger `nfsw` filter
														# at the cost of increasing the possibility of filtering benign images
														snake_case_   : int						=			0.0

														for concept_idx in range(len(special_cos_dist[0]   )   ):
																				snake_case_   : List[str]						=			special_cos_dist[i][concept_idx]
																				snake_case_   : Union[str, Any]						=			self.special_care_embeds_weights[concept_idx].item()
																				snake_case_   : Tuple						=			round(concept_cos - concept_threshold + adjustment							,    3   )
																				if result_img["special_scores"][concept_idx] > 0:
																										result_img["special_care"].append({concept_idx, result_img["""special_scores"""][concept_idx]}   )
																										snake_case_   : Dict						=			0.01

														for concept_idx in range(len(cos_dist[0]   )   ):
																				snake_case_   : int						=			cos_dist[i][concept_idx]
																				snake_case_   : List[Any]						=			self.concept_embeds_weights[concept_idx].item()
																				snake_case_   : List[str]						=			round(concept_cos - concept_threshold + adjustment							,    3   )
																				if result_img["concept_scores"][concept_idx] > 0:
																										result_img["bad_concepts"].append(_lowercase   )

														result.append(_lowercase   )

								snake_case_   : Union[str, Any]						=			[len(res["""bad_concepts"""]   ) > 0 for res in result]

								return images, has_nsfw_concepts





		@torch.no_grad()
		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase   ) ->						Tuple:







								'''simple docstring'''

								snake_case_   : Optional[Any]						=			self.vision_model(_lowercase   )[1]  # pooled_output
								snake_case_   : List[str]						=			self.visual_projection(_lowercase   )

								snake_case_   : str						=			cosine_distance(_lowercase							,    self.special_care_embeds   )
								snake_case_   : Optional[int]						=			cosine_distance(_lowercase							,    self.concept_embeds   )

								# increase this value to create a stronger `nsfw` filter
								# at the cost of increasing the possibility of filtering benign images
								snake_case_   : Tuple						=			0.0

								snake_case_   : List[Any]						=			special_cos_dist - self.special_care_embeds_weights + adjustment
								# special_scores = special_scores.round(decimals=3)
								snake_case_   : str						=			torch.any(special_scores > 0							,    dim=1   )
								snake_case_   : List[str]						=			special_care * 0.01
								snake_case_   : Optional[int]						=			special_adjustment.unsqueeze(1   ).expand(-1							,    cos_dist.shape[1]   )

								snake_case_   : Optional[Any]						=			(cos_dist - self.concept_embeds_weights) + special_adjustment
								# concept_scores = concept_scores.round(decimals=3)
								snake_case_   : str						=			torch.any(concept_scores > 0							,    dim=1   )

								return images, has_nsfw_concepts

