






import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A						:							Tuple							=  logging.get_logger(__name__)

__A						:							List[str]							=  {
    """google/pix2struct-textcaps-base""": (
        """https://huggingface.co/google/pix2struct-textcaps-base/resolve/main/config.json"""
    ),
}




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     '''pix2struct_text_model'''
							a__						  =     ['''past_key_values''']
							a__						  =     {
							    '''hidden_size''': '''hidden_size''',
							    '''num_attention_heads''': '''num_heads''',
							    '''num_hidden_layers''': '''num_layers''',
							}





							def __init__(					self       :	Any       ,     a       :	str=50_244       ,     a       :	Optional[int]=768       ,     a       :	int=64       ,     a       :	Any=2_048       ,     a       :	List[str]=12       ,     a       :	Optional[int]=12       ,     a       :	List[str]=32       ,     a       :	Union[str, Any]=128       ,     a       :	Any=0.1       ,     a       :	str=1E-6       ,     a       :	int=1.0       ,     a       :	Optional[int]="gelu_new"       ,     a       :	List[Any]=0       ,     a       :	Dict=False       ,     a       :	Union[str, Any]=0       ,     a       :	Union[str, Any]=1       ,     a       :	List[Any]=False       ,     a       :	List[Any]=True       ,     **a       :	Optional[int]       ,     )			->					Dict:
										SCREAMING_SNAKE_CASE           =					vocab_size
										SCREAMING_SNAKE_CASE           =					hidden_size
										SCREAMING_SNAKE_CASE           =					d_kv
										SCREAMING_SNAKE_CASE           =					d_ff
										SCREAMING_SNAKE_CASE           =					num_layers
										SCREAMING_SNAKE_CASE           =					num_heads
										SCREAMING_SNAKE_CASE           =					relative_attention_num_buckets
										SCREAMING_SNAKE_CASE           =					relative_attention_max_distance
										SCREAMING_SNAKE_CASE           =					dropout_rate
										SCREAMING_SNAKE_CASE           =					layer_norm_epsilon
										SCREAMING_SNAKE_CASE           =					initializer_factor
										SCREAMING_SNAKE_CASE           =					use_cache

										SCREAMING_SNAKE_CASE           =					eos_token_id
										SCREAMING_SNAKE_CASE           =					decoder_start_token_id

										# for backwards compatibility
										SCREAMING_SNAKE_CASE           =					dense_act_fn

										super().__init__(
										    pad_token_id=a       ,     eos_token_id=a       ,     decoder_start_token_id=a       ,     tie_word_embeddings=a       ,     is_decoder=a       ,     **a       ,     )







							@classmethod
							def    _UpperCAmelCase     (					cls       :	Tuple       ,     a       :	Union[str, os.PathLike]       ,     **a       :	Optional[Any]			)			->					"PretrainedConfig":
										cls._set_token_in_kwargs(a			)

										SCREAMING_SNAKE_CASE     ,  SCREAMING_SNAKE_CASE           =					cls.get_config_dict(a       ,     **a			)

										# get the text config dict if we are loading from Pix2StructConfig
										if config_dict.get("""model_type"""			) == "pix2struct":
													SCREAMING_SNAKE_CASE           =					config_dict["""text_config"""]

										if "model_type" in config_dict and hasattr(cls       ,     """model_type"""			) and config_dict["model_type"] != cls.model_type:
													logger.warning(
													    f"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
													    f"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""			)

										return cls.from_dict(a       ,     **a			)





class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     '''pix2struct_vision_model'''





							def __init__(					self       :	Optional[Any]       ,     a       :	int=768       ,     a       :	Optional[int]=768       ,     a       :	int=2_048       ,     a       :	List[Any]=64       ,     a       :	str=12       ,     a       :	Any=12       ,     a       :	Optional[int]="gelu_new"       ,     a       :	List[Any]=1E-6       ,     a       :	Optional[int]=0.0       ,     a       :	Dict=0.0       ,     a       :	Any=1E-1_0       ,     a       :	str=1.0       ,     a       :	Optional[int]=4_096       ,     a       :	Any=32       ,     a       :	Optional[int]=128       ,     **a       :	Tuple       ,     )			->					Optional[Any]:
										super().__init__(**a			)

										SCREAMING_SNAKE_CASE           =					hidden_size
										SCREAMING_SNAKE_CASE           =					patch_embed_hidden_size
										SCREAMING_SNAKE_CASE           =					d_ff
										SCREAMING_SNAKE_CASE           =					dropout_rate
										SCREAMING_SNAKE_CASE           =					num_hidden_layers
										SCREAMING_SNAKE_CASE           =					num_attention_heads
										SCREAMING_SNAKE_CASE           =					initializer_range
										SCREAMING_SNAKE_CASE           =					initializer_factor
										SCREAMING_SNAKE_CASE           =					attention_dropout
										SCREAMING_SNAKE_CASE           =					layer_norm_eps
										SCREAMING_SNAKE_CASE           =					dense_act_fn
										SCREAMING_SNAKE_CASE           =					seq_len
										SCREAMING_SNAKE_CASE           =					relative_attention_num_buckets
										SCREAMING_SNAKE_CASE           =					relative_attention_max_distance
										SCREAMING_SNAKE_CASE           =					d_kv







							@classmethod
							def    _UpperCAmelCase     (					cls       :	int       ,     a       :	Union[str, os.PathLike]       ,     **a       :	Any			)			->					"PretrainedConfig":
										cls._set_token_in_kwargs(a			)

										SCREAMING_SNAKE_CASE     ,  SCREAMING_SNAKE_CASE           =					cls.get_config_dict(a       ,     **a			)

										# get the vision config dict if we are loading from Pix2StructConfig
										if config_dict.get("""model_type"""			) == "pix2struct":
													SCREAMING_SNAKE_CASE           =					config_dict["""vision_config"""]

										if "model_type" in config_dict and hasattr(cls       ,     """model_type"""			) and config_dict["model_type"] != cls.model_type:
													logger.warning(
													    f"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
													    f"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""			)

										return cls.from_dict(a       ,     **a			)





class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     '''pix2struct'''
							a__						  =     True





							def __init__(					self       :	Tuple       ,     a       :	List[str]=None       ,     a       :	int=None       ,     a       :	Union[str, Any]=1.0       ,     a       :	Optional[int]=0.02       ,     a       :	int=False       ,     a       :	List[str]=False       ,     a       :	Any=True       ,     **a       :	Optional[Any]       ,     )			->					List[str]:
										super().__init__(tie_word_embeddings=a       ,     is_encoder_decoder=a       ,     **a			)

										if text_config is None:
													SCREAMING_SNAKE_CASE           =					{}
													logger.info("""text_config is None. Initializing the Pix2StructTextConfig with default values."""			)

										if vision_config is None:
													SCREAMING_SNAKE_CASE           =					{}
													logger.info("""vision_config is None. Initializing the Pix2StructVisionConfig with default values."""			)

										SCREAMING_SNAKE_CASE           =					PixaStructTextConfig(**a			)
										SCREAMING_SNAKE_CASE           =					PixaStructVisionConfig(**a			)

										SCREAMING_SNAKE_CASE           =					self.text_config.decoder_start_token_id
										SCREAMING_SNAKE_CASE           =					self.text_config.pad_token_id
										SCREAMING_SNAKE_CASE           =					self.text_config.eos_token_id

										SCREAMING_SNAKE_CASE           =					initializer_factor
										SCREAMING_SNAKE_CASE           =					initializer_range

										SCREAMING_SNAKE_CASE           =					self.initializer_range
										SCREAMING_SNAKE_CASE           =					self.initializer_range

										SCREAMING_SNAKE_CASE           =					is_vqa





							@classmethod
							def    _UpperCAmelCase     (					cls       :	Optional[int]       ,     a       :	PixaStructTextConfig       ,     a       :	PixaStructVisionConfig       ,     **a       :	Dict			)			->					Optional[int]:
										return cls(text_config=text_config.to_dict()       ,     vision_config=vision_config.to_dict()       ,     **a			)







							def    _UpperCAmelCase     (					self       :	List[str]			)			->					Any:
										SCREAMING_SNAKE_CASE           =					copy.deepcopy(self.__dict__			)
										SCREAMING_SNAKE_CASE           =					self.text_config.to_dict()
										SCREAMING_SNAKE_CASE           =					self.vision_config.to_dict()
										SCREAMING_SNAKE_CASE           =					self.__class__.model_type
										return output












import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__A						:							Union[str, Any]							=  logging.get_logger(__name__)

__A						:							str							=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__A						:							List[Any]							=  {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

__A						:							Tuple							=  {
    """Salesforce/codegen-350M-mono""": 2_0_4_8,
}




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     VOCAB_FILES_NAMES
							a__						  =     PRETRAINED_VOCAB_FILES_MAP
							a__						  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a__						  =     ['''input_ids''', '''attention_mask''']
							a__						  =     CodeGenTokenizer





							def __init__(					self       :	Tuple       ,     a       :	Dict=None       ,     a       :	Tuple=None       ,     a       :	Tuple=None       ,     a       :	Optional[Any]="<|endoftext|>"       ,     a       :	Any="<|endoftext|>"       ,     a       :	Dict="<|endoftext|>"       ,     a       :	Optional[int]=False       ,     **a       :	Tuple       ,     )			->					str:
										super().__init__(
										    a       ,     a       ,     tokenizer_file=a       ,     unk_token=a       ,     bos_token=a       ,     eos_token=a       ,     add_prefix_space=a       ,     **a       ,     )

										if kwargs.pop("""add_bos_token"""       ,     a			):
													SCREAMING_SNAKE_CASE           =					kwargs.pop("""name_or_path"""       ,     """"""			)
													raise ValueError(
													    """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
													    """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
													    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
													    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
													    """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
													    """ so that the fast tokenizer works correctly."""			)

										SCREAMING_SNAKE_CASE           =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
										if pre_tok_state.get("""add_prefix_space"""       ,     a			) != add_prefix_space:
													SCREAMING_SNAKE_CASE           =					getattr(a       ,     pre_tok_state.pop("""type"""			)			)
													SCREAMING_SNAKE_CASE           =					add_prefix_space
													SCREAMING_SNAKE_CASE           =					pre_tok_class(**a			)

										SCREAMING_SNAKE_CASE           =					add_prefix_space





							def    _UpperCAmelCase     (					self       :	Optional[Any]       ,     *a       :	Optional[Any]       ,     **a       :	List[Any]			)			->					BatchEncoding:
										SCREAMING_SNAKE_CASE           =					kwargs.get("""is_split_into_words"""       ,     a			)
										assert self.add_prefix_space or not is_split_into_words, (
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*a       ,     **a			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     *a       :	Tuple       ,     **a       :	List[Any]			)			->					BatchEncoding:
										SCREAMING_SNAKE_CASE           =					kwargs.get("""is_split_into_words"""       ,     a			)

										assert self.add_prefix_space or not is_split_into_words, (
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*a       ,     **a			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	str       ,     a       :	Optional[str] = None			)			->					Tuple[str]:
										SCREAMING_SNAKE_CASE           =					self._tokenizer.model.save(a       ,     name=a			)
										return tuple(a			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]       ,     a       :	bool = False       ,     a       :	bool = None       ,     a       :	Optional[List[str]] = None       ,     **a       :	List[Any]       ,     )			->					str:
										SCREAMING_SNAKE_CASE           =					super().decode(
										    token_ids=a       ,     skip_special_tokens=a       ,     clean_up_tokenization_spaces=a       ,     **a       ,     )

										if truncate_before_pattern is not None and len(a			) > 0:
													SCREAMING_SNAKE_CASE           =					self.truncate(a       ,     a			)

										return decoded_text







							def    _UpperCAmelCase     (					self       :	Any       ,     a       :	Optional[Any]       ,     a       :	List[str]			)			->					Any:
										def find_re(a       :	Tuple       ,     a       :	Any       ,     a       :	List[str]			):
													SCREAMING_SNAKE_CASE           =					pattern.search(a       ,     a			)
													return m.start() if m else -1

										SCREAMING_SNAKE_CASE           =					[re.compile(a       ,     re.MULTILINE			) for pattern in truncate_before_pattern]

										SCREAMING_SNAKE_CASE           =					list(re.finditer("""^print"""       ,     a       ,     re.MULTILINE			)			)

										if len(a			) > 1:
													SCREAMING_SNAKE_CASE           =					completion[: prints[1].start()]

										SCREAMING_SNAKE_CASE           =					list(re.finditer("""^def"""       ,     a       ,     re.MULTILINE			)			)

										if len(a			) > 1:
													SCREAMING_SNAKE_CASE           =					completion[: defs[1].start()]

										SCREAMING_SNAKE_CASE           =					0

										SCREAMING_SNAKE_CASE           =					[
										    pos for pos in [find_re(a       ,     a       ,     a			) for terminal in terminals] if pos != -1
										]

										if len(a			) > 0:
													return completion[: min(a			)]
										else:
													return completion






