






'''simple docstring'''




import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


lowerCAmelCase__          =						logging.get_logger(__name__)


lowerCAmelCase__          =						{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase__          =						{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase__          =						{
    'allenai/led-base-16384': 1_6384,
}







class       __lowercase					(__lowerCamelCase    ):
	_lowerCamelCase              =  VOCAB_FILES_NAMES
	_lowerCamelCase              =  PRETRAINED_VOCAB_FILES_MAP
	_lowerCamelCase              =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCamelCase              =  LEDTokenizer
	_lowerCamelCase              =  ['''input_ids''', '''attention_mask''']

	def __init__(      self   :						Tuple  ,					UpperCAmelCase_   :						int=None  ,					UpperCAmelCase_   :						Dict=None  ,					UpperCAmelCase_   :						str=None  ,					UpperCAmelCase_   :						Union[str, Any]="replace"  ,					UpperCAmelCase_   :						str="<s>"  ,					UpperCAmelCase_   :						Tuple="</s>"  ,					UpperCAmelCase_   :						Any="</s>"  ,					UpperCAmelCase_   :						Tuple="<s>"  ,					UpperCAmelCase_   :						Optional[int]="<unk>"  ,					UpperCAmelCase_   :						Optional[Any]="<pad>"  ,					UpperCAmelCase_   :						Tuple="<mask>"  ,					UpperCAmelCase_   :						Union[str, Any]=False  ,					UpperCAmelCase_   :						Any=True  ,					**UpperCAmelCase_   :						List[Any]  ,					):
			super().__init__(
			    UpperCAmelCase_  ,					UpperCAmelCase_  ,					tokenizer_file=UpperCAmelCase_  ,					errors=UpperCAmelCase_  ,					bos_token=UpperCAmelCase_  ,					eos_token=UpperCAmelCase_  ,					sep_token=UpperCAmelCase_  ,					cls_token=UpperCAmelCase_  ,					unk_token=UpperCAmelCase_  ,					pad_token=UpperCAmelCase_  ,					mask_token=UpperCAmelCase_  ,					add_prefix_space=UpperCAmelCase_  ,					trim_offsets=UpperCAmelCase_  ,					**UpperCAmelCase_  ,					)

			UpperCamelCase__				:					List[Any]					=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
			if pre_tok_state.get('add_prefix_space'  ,					UpperCAmelCase_) != add_prefix_space:
					UpperCamelCase__				:					Any					=					getattr(UpperCAmelCase_  ,					pre_tok_state.pop('type'))
					UpperCamelCase__				:					Optional[int]					=					add_prefix_space
					UpperCamelCase__				:					Union[str, Any]					=					pre_tok_class(**UpperCAmelCase_)

			UpperCamelCase__				:					Optional[int]					=					add_prefix_space

			# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
			UpperCamelCase__				:					str					=					'post_processor'
			UpperCamelCase__				:					str					=					getattr(self.backend_tokenizer  ,					UpperCAmelCase_  ,					UpperCAmelCase_)
			if tokenizer_component_instance:
					UpperCamelCase__				:					Optional[int]					=					json.loads(tokenizer_component_instance.__getstate__())

					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
					if "sep" in state:
							UpperCamelCase__				:					int					=					tuple(state['sep'])
					if "cls" in state:
							UpperCamelCase__				:					Tuple					=					tuple(state['cls'])

					UpperCamelCase__				:					Optional[Any]					=					False

					if state.get('add_prefix_space'  ,					UpperCAmelCase_) != add_prefix_space:
							UpperCamelCase__				:					int					=					add_prefix_space
							UpperCamelCase__				:					Union[str, Any]					=					True

					if state.get('trim_offsets'  ,					UpperCAmelCase_) != trim_offsets:
							UpperCamelCase__				:					List[str]					=					trim_offsets
							UpperCamelCase__				:					Tuple					=					True

					if changes_to_apply:
							UpperCamelCase__				:					List[str]					=					getattr(UpperCAmelCase_  ,					state.pop('type'))
							UpperCamelCase__				:					Any					=					component_class(**UpperCAmelCase_)
							setattr(self.backend_tokenizer  ,					UpperCAmelCase_  ,					UpperCAmelCase_)

	@property
	# Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
	def      __UpperCamelCase							(      self   :						Dict):
			if self._mask_token is None:
					if self.verbose:
							logger.error('Using mask_token, but it is not set yet.')
					return None
			return str(self._mask_token)

	@mask_token.setter
	def      __UpperCamelCase							(      self   :						Union[str, Any]  ,					UpperCAmelCase_   :						List[Any]):
			UpperCamelCase__				:					str					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else value
			UpperCamelCase__				:					Optional[int]					=					value

	def      __UpperCamelCase							(      self   :						List[Any]  ,					*UpperCAmelCase_   :						Optional[int]  ,					**UpperCAmelCase_   :						List[str]):
			UpperCamelCase__				:					str					=					kwargs.get('is_split_into_words'  ,					UpperCAmelCase_)

			if is_split_into_words and not self.add_prefix_space:
					raise ValueError(
					    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
					    'to use it with pretokenized inputs.')

			return super()._batch_encode_plus(*UpperCAmelCase_  ,					**UpperCAmelCase_)

	def      __UpperCamelCase							(      self   :						List[str]  ,					*UpperCAmelCase_   :						Dict  ,					**UpperCAmelCase_   :						Optional[int]):
			UpperCamelCase__				:					List[str]					=					kwargs.get('is_split_into_words'  ,					UpperCAmelCase_)

			if is_split_into_words and not self.add_prefix_space:
					raise ValueError(
					    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
					    'to use it with pretokenized inputs.')

			return super()._encode_plus(*UpperCAmelCase_  ,					**UpperCAmelCase_)

	def      __UpperCamelCase							(      self   :						List[str]  ,					UpperCAmelCase_   :						int  ,					UpperCAmelCase_   :						Any = None):
			UpperCamelCase__				:					List[str]					=					self._tokenizer.model.save(UpperCAmelCase_  ,					name=UpperCAmelCase_)
			return tuple(UpperCAmelCase_)

	def      __UpperCamelCase							(      self   :						Dict  ,					UpperCAmelCase_   :						int  ,					UpperCAmelCase_   :						List[str]=None):
			UpperCamelCase__				:					Optional[Any]					=					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
			if token_ids_a is None:
					return output

			return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

	def      __UpperCamelCase							(      self   :						Optional[Any]  ,					UpperCAmelCase_   :						Dict  ,					UpperCAmelCase_   :						Tuple = None):
			UpperCamelCase__				:					Optional[Any]					=					[self.sep_token_id]
			UpperCamelCase__				:					List[Any]					=					[self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





	def      __UpperCamelCase							(      self   :						Dict  ,					UpperCAmelCase_   :						Tuple  ,					UpperCAmelCase_   :						str = None  ,					UpperCAmelCase_   :						Union[str, Any] = PaddingStrategy.DO_NOT_PAD  ,					UpperCAmelCase_   :						str = None  ,					UpperCAmelCase_   :						int = None  ,					):
			UpperCamelCase__				:					int					=					super()._pad(
			    encoded_inputs=UpperCAmelCase_  ,					max_length=UpperCAmelCase_  ,					padding_strategy=UpperCAmelCase_  ,					pad_to_multiple_of=UpperCAmelCase_  ,					return_attention_mask=UpperCAmelCase_  ,					)

			# Load from model defaults
			if return_attention_mask is None:
					UpperCamelCase__				:					Union[str, Any]					=					'attention_mask' in self.model_input_names

			if return_attention_mask and "global_attention_mask" in encoded_inputs:
					UpperCamelCase__				:					Dict					=					encoded_inputs[self.model_input_names[0]]
					# `global_attention_mask` need to have the same length as other (sequential) inputs.
					UpperCamelCase__				:					Union[str, Any]					=					len(encoded_inputs['global_attention_mask']) != len(UpperCAmelCase_)

					if needs_to_be_padded:
							UpperCamelCase__				:					int					=					len(UpperCAmelCase_) - len(encoded_inputs['global_attention_mask'])

							if self.padding_side == "right":
									# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
									UpperCamelCase__				:					List[str]					=					(
									    encoded_inputs['global_attention_mask'] + [-1] * difference
									)
							elif self.padding_side == "left":
									UpperCamelCase__				:					Any					=					[-1] * difference + encoded_inputs[
									    'global_attention_mask'
									]
							else:
									raise ValueError('Invalid padding strategy:' + str(self.padding_side))

			return encoded_inputs












'''simple docstring'''




import warnings
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase__          =						logging.get_logger(__name__)

lowerCAmelCase__          =						{
    'nvidia/segformer-b0-finetuned-ade-512-512': (
        'https://huggingface.co/nvidia/segformer-b0-finetuned-ade-512-512/resolve/main/config.json'
    ),
    # See all SegFormer models at https://huggingface.co/models?filter=segformer
}







class       __lowercase					(__lowerCamelCase    ):
	_lowerCamelCase              =  '''segformer'''

	def __init__(      self   :						Tuple  ,					UpperCAmelCase_   :						Optional[Any]=3  ,					UpperCAmelCase_   :						Optional[int]=4  ,					UpperCAmelCase_   :						Tuple=[2, 2, 2, 2]  ,					UpperCAmelCase_   :						List[str]=[8, 4, 2, 1]  ,					UpperCAmelCase_   :						Union[str, Any]=[32, 64, 160, 256]  ,					UpperCAmelCase_   :						Any=[7, 3, 3, 3]  ,					UpperCAmelCase_   :						Any=[4, 2, 2, 2]  ,					UpperCAmelCase_   :						Union[str, Any]=[1, 2, 5, 8]  ,					UpperCAmelCase_   :						Tuple=[4, 4, 4, 4]  ,					UpperCAmelCase_   :						str="gelu"  ,					UpperCAmelCase_   :						List[Any]=0.0  ,					UpperCAmelCase_   :						int=0.0  ,					UpperCAmelCase_   :						int=0.1  ,					UpperCAmelCase_   :						List[str]=0.02  ,					UpperCAmelCase_   :						Dict=0.1  ,					UpperCAmelCase_   :						Dict=1e-6  ,					UpperCAmelCase_   :						int=256  ,					UpperCAmelCase_   :						Optional[int]=255  ,					**UpperCAmelCase_   :						Tuple  ,					):
			super().__init__(**UpperCAmelCase_)

			if "reshape_last_stage" in kwargs and kwargs["reshape_last_stage"] is False:
					warnings.warn(
					    'Reshape_last_stage is set to False in this config. This argument is deprecated and will soon be'
					    ' removed, as the behaviour will default to that of reshape_last_stage = True.'  ,					UpperCAmelCase_  ,					)

			UpperCamelCase__				:					List[Any]					=					num_channels
			UpperCamelCase__				:					Any					=					num_encoder_blocks
			UpperCamelCase__				:					Dict					=					depths
			UpperCamelCase__				:					int					=					sr_ratios
			UpperCamelCase__				:					str					=					hidden_sizes
			UpperCamelCase__				:					List[str]					=					patch_sizes
			UpperCamelCase__				:					Optional[int]					=					strides
			UpperCamelCase__				:					Dict					=					mlp_ratios
			UpperCamelCase__				:					List[str]					=					num_attention_heads
			UpperCamelCase__				:					int					=					hidden_act
			UpperCamelCase__				:					Any					=					hidden_dropout_prob
			UpperCamelCase__				:					str					=					attention_probs_dropout_prob
			UpperCamelCase__				:					List[str]					=					classifier_dropout_prob
			UpperCamelCase__				:					List[Any]					=					initializer_range
			UpperCamelCase__				:					Union[str, Any]					=					drop_path_rate
			UpperCamelCase__				:					int					=					layer_norm_eps
			UpperCamelCase__				:					Dict					=					decoder_hidden_size
			UpperCamelCase__				:					List[Any]					=					kwargs.get('reshape_last_stage'  ,					UpperCAmelCase_)
			UpperCamelCase__				:					List[str]					=					semantic_loss_ignore_index







class       __lowercase					(__lowerCamelCase    ):
	_lowerCamelCase              =  version.parse('''1.11'''    )

	@property
	def      __UpperCamelCase							(      self   :						Optional[Any]):
			return OrderedDict(
			    [
			        ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
			    ])

	@property
	def      __UpperCamelCase							(      self   :						Optional[Any]):
			return 1e-4





	@property
	def      __UpperCamelCase							(      self   :						Any):
			return 12




