






'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__          =						logging.get_logger(__name__)

lowerCAmelCase__          =						{
    'google/pegasus-large': 'https://huggingface.co/google/pegasus-large/resolve/main/config.json',
    # See all PEGASUS models at https://huggingface.co/models?filter=pegasus
}







class       __lowercase					(__lowerCamelCase    ):
	_lowerCamelCase              =  '''pegasus'''
	_lowerCamelCase              =  ['''past_key_values''']
	_lowerCamelCase              =  {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}

	def __init__(      self   :						int  ,					UpperCAmelCase_   :						Optional[int]=50_265  ,					UpperCAmelCase_   :						Optional[Any]=1_024  ,					UpperCAmelCase_   :						int=12  ,					UpperCAmelCase_   :						Optional[int]=4_096  ,					UpperCAmelCase_   :						Tuple=16  ,					UpperCAmelCase_   :						Optional[int]=12  ,					UpperCAmelCase_   :						Dict=4_096  ,					UpperCAmelCase_   :						Optional[Any]=16  ,					UpperCAmelCase_   :						Tuple=0.0  ,					UpperCAmelCase_   :						List[str]=0.0  ,					UpperCAmelCase_   :						Optional[int]=True  ,					UpperCAmelCase_   :						Union[str, Any]=True  ,					UpperCAmelCase_   :						List[Any]="gelu"  ,					UpperCAmelCase_   :						Any=1_024  ,					UpperCAmelCase_   :						Optional[int]=0.1  ,					UpperCAmelCase_   :						Union[str, Any]=0.0  ,					UpperCAmelCase_   :						List[Any]=0.0  ,					UpperCAmelCase_   :						Tuple=0.02  ,					UpperCAmelCase_   :						List[str]=0  ,					UpperCAmelCase_   :						str=False  ,					UpperCAmelCase_   :						int=0  ,					UpperCAmelCase_   :						Optional[int]=1  ,					UpperCAmelCase_   :						Union[str, Any]=1  ,					**UpperCAmelCase_   :						List[str]  ,					):
			UpperCamelCase__				:					Optional[int]					=					vocab_size
			UpperCamelCase__				:					Union[str, Any]					=					max_position_embeddings
			UpperCamelCase__				:					List[Any]					=					d_model
			UpperCamelCase__				:					Tuple					=					encoder_ffn_dim
			UpperCamelCase__				:					str					=					encoder_layers
			UpperCamelCase__				:					str					=					encoder_attention_heads
			UpperCamelCase__				:					Optional[int]					=					decoder_ffn_dim
			UpperCamelCase__				:					Dict					=					decoder_layers
			UpperCamelCase__				:					Any					=					decoder_attention_heads
			UpperCamelCase__				:					Dict					=					dropout
			UpperCamelCase__				:					List[Any]					=					attention_dropout
			UpperCamelCase__				:					List[Any]					=					activation_dropout
			UpperCamelCase__				:					Union[str, Any]					=					activation_function
			UpperCamelCase__				:					str					=					init_std
			UpperCamelCase__				:					Union[str, Any]					=					encoder_layerdrop
			UpperCamelCase__				:					List[Any]					=					decoder_layerdrop
			UpperCamelCase__				:					Optional[Any]					=					use_cache
			UpperCamelCase__				:					Any					=					encoder_layers
			UpperCamelCase__				:					Union[str, Any]					=					scale_embedding  # scale factor will be sqrt(d_model) if True
			super().__init__(
			    pad_token_id=UpperCAmelCase_  ,					eos_token_id=UpperCAmelCase_  ,					is_encoder_decoder=UpperCAmelCase_  ,					decoder_start_token_id=UpperCAmelCase_  ,					forced_eos_token_id=UpperCAmelCase_  ,					**UpperCAmelCase_  ,					)

	@property
	def      __UpperCamelCase							(      self   :						Dict):
			return self.encoder_attention_heads





	@property
	def      __UpperCamelCase							(      self   :						List[Any]):
			return self.d_model








'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation

lowerCAmelCase__          =						logging.get_logger(__name__)


lowerCAmelCase__          =						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

lowerCAmelCase__          =						{
    'vocab_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'},
    'merges_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'},
    'tokenizer_config_file': {
        'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'
    },
}

lowerCAmelCase__          =						{'facebook/blenderbot-3B': 128}







@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def       __UpperCAmelCase						(  )							->     Union[str, Any]:
		UpperCamelCase__				:					Optional[Any]					=					(
		    list(range(ord('!')     ,      ord('~') + 1)) + list(range(ord('¡')     ,      ord('¬') + 1)) + list(range(ord('®')     ,      ord('ÿ') + 1))
		)
		UpperCamelCase__				:					List[Any]					=					bs[:]
		UpperCamelCase__				:					Optional[int]					=					0
		for b in range(2**8):
				if b not in bs:
						bs.append(lowerCamelCase_)
						cs.append(2**8 + n)
						n += 1
		UpperCamelCase__				:					Union[str, Any]					=					[chr(lowerCamelCase_) for n in cs]
		return dict(zip(lowerCamelCase_     ,      lowerCamelCase_))







def       __UpperCAmelCase						(  lowerCamelCase_)							->     Tuple:
		UpperCamelCase__				:					Any					=					set()
		UpperCamelCase__				:					Dict					=					word[0]
		for char in word[1:]:
				pairs.add((prev_char, char))
				UpperCamelCase__				:					str					=					char
		return pairs







class       __lowercase					(__lowerCamelCase    ):
	_lowerCamelCase              =  VOCAB_FILES_NAMES
	_lowerCamelCase              =  PRETRAINED_VOCAB_FILES_MAP
	_lowerCamelCase              =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCamelCase              =  ['''input_ids''', '''attention_mask''']

	def __init__(      self   :						Tuple  ,					UpperCAmelCase_   :						Tuple  ,					UpperCAmelCase_   :						Tuple  ,					UpperCAmelCase_   :						Dict="replace"  ,					UpperCAmelCase_   :						int="<s>"  ,					UpperCAmelCase_   :						Tuple="</s>"  ,					UpperCAmelCase_   :						Any="</s>"  ,					UpperCAmelCase_   :						List[Any]="<s>"  ,					UpperCAmelCase_   :						List[str]="<unk>"  ,					UpperCAmelCase_   :						Any="<pad>"  ,					UpperCAmelCase_   :						Optional[Any]="<mask>"  ,					UpperCAmelCase_   :						str=False  ,					**UpperCAmelCase_   :						List[Any]  ,					):
			UpperCamelCase__				:					Union[str, Any]					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else bos_token
			UpperCamelCase__				:					List[str]					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else eos_token
			UpperCamelCase__				:					Optional[Any]					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else sep_token
			UpperCamelCase__				:					int					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else cls_token
			UpperCamelCase__				:					Tuple					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else unk_token
			UpperCamelCase__				:					Optional[Any]					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else pad_token

			# Mask token behave like a normal word, i.e. include the space before it
			UpperCamelCase__				:					Any					=					AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_) else mask_token

			super().__init__(
			    errors=UpperCAmelCase_  ,					bos_token=UpperCAmelCase_  ,					eos_token=UpperCAmelCase_  ,					unk_token=UpperCAmelCase_  ,					sep_token=UpperCAmelCase_  ,					cls_token=UpperCAmelCase_  ,					pad_token=UpperCAmelCase_  ,					mask_token=UpperCAmelCase_  ,					add_prefix_space=UpperCAmelCase_  ,					**UpperCAmelCase_  ,					)

			with open(UpperCAmelCase_  ,					encoding='utf-8') as vocab_handle:
					UpperCamelCase__				:					Any					=					json.load(UpperCAmelCase_)
			UpperCamelCase__				:					Dict					=					{v: k for k, v in self.encoder.items()}
			UpperCamelCase__				:					Any					=					errors  # how to handle errors in decoding
			UpperCamelCase__				:					Tuple					=					bytes_to_unicode()
			UpperCamelCase__				:					Union[str, Any]					=					{v: k for k, v in self.byte_encoder.items()}
			with open(UpperCAmelCase_  ,					encoding='utf-8') as merges_handle:
					UpperCamelCase__				:					List[Any]					=					merges_handle.read().split('\n')[1:-1]
			UpperCamelCase__				:					List[Any]					=					[tuple(merge.split()) for merge in bpe_merges]
			UpperCamelCase__				:					Any					=					dict(zip(UpperCAmelCase_  ,					range(len(UpperCAmelCase_))))
			UpperCamelCase__				:					Dict					=					{}
			UpperCamelCase__				:					Dict					=					add_prefix_space

			# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
			UpperCamelCase__				:					Any					=					re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')

	@property
	# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
	def      __UpperCamelCase							(      self   :						Tuple):
			return len(self.encoder)

	def      __UpperCamelCase							(      self   :						Tuple):
			return dict(self.encoder  ,					**self.added_tokens_encoder)

	def      __UpperCamelCase							(      self   :						List[Any]  ,					UpperCAmelCase_   :						Union[str, Any]):
			if token in self.cache:
					return self.cache[token]
			UpperCamelCase__				:					Optional[int]					=					tuple(UpperCAmelCase_)
			UpperCamelCase__				:					int					=					get_pairs(UpperCAmelCase_)

			if not pairs:
					return token

			while True:
					UpperCamelCase__				:					Tuple					=					min(UpperCAmelCase_  ,					key=lambda UpperCAmelCase_: self.bpe_ranks.get(UpperCAmelCase_  ,					float('inf')))
					if bigram not in self.bpe_ranks:
							break
					UpperCamelCase__,	UpperCamelCase__				:					Tuple					=					bigram
					UpperCamelCase__				:					Dict					=					[]
					UpperCamelCase__				:					Optional[int]					=					0
					while i < len(UpperCAmelCase_):
							try:
									UpperCamelCase__				:					Tuple					=					word.index(UpperCAmelCase_  ,					UpperCAmelCase_)
							except ValueError:
									new_word.extend(word[i:])
									break
							else:
									new_word.extend(word[i:j])
									UpperCamelCase__				:					Any					=					j

							if word[i] == first and i < len(UpperCAmelCase_) - 1 and word[i + 1] == second:
									new_word.append(first + second)
									i += 2
							else:
									new_word.append(word[i])
									i += 1
					UpperCamelCase__				:					List[str]					=					tuple(UpperCAmelCase_)
					UpperCamelCase__				:					Dict					=					new_word
					if len(UpperCAmelCase_) == 1:
							break
					else:
							UpperCamelCase__				:					Optional[int]					=					get_pairs(UpperCAmelCase_)
			UpperCamelCase__				:					Optional[Any]					=					' '.join(UpperCAmelCase_)
			UpperCamelCase__				:					List[Any]					=					word
			return word

	def      __UpperCamelCase							(      self   :						List[str]  ,					UpperCAmelCase_   :						Any):
			UpperCamelCase__				:					Optional[Any]					=					[]
			for token in re.findall(self.pat  ,					UpperCAmelCase_):
					UpperCamelCase__				:					Optional[int]					=					''.join(
					    self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase_).split(' '))
			return bpe_tokens

	def      __UpperCamelCase							(      self   :						Union[str, Any]  ,					UpperCAmelCase_   :						Optional[Any]):
			return self.encoder.get(UpperCAmelCase_  ,					self.encoder.get(self.unk_token))

	def      __UpperCamelCase							(      self   :						Any  ,					UpperCAmelCase_   :						Optional[int]):
			return self.decoder.get(UpperCAmelCase_)

	def      __UpperCamelCase							(      self   :						List[Any]  ,					UpperCAmelCase_   :						int):
			UpperCamelCase__				:					int					=					''.join(UpperCAmelCase_)
			UpperCamelCase__				:					Any					=					bytearray([self.byte_decoder[c] for c in text]).decode('utf-8'  ,					errors=self.errors)
			return text

	def      __UpperCamelCase							(      self   :						str  ,					UpperCAmelCase_   :						str  ,					UpperCAmelCase_   :						Optional[str] = None):
			if not os.path.isdir(UpperCAmelCase_):
					logger.error(F'Vocabulary path ({save_directory}) should be a directory')
					return
			UpperCamelCase__				:					str					=					os.path.join(
			    UpperCAmelCase_  ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
			UpperCamelCase__				:					Optional[Any]					=					os.path.join(
			    UpperCAmelCase_  ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

			with open(UpperCAmelCase_  ,					'w'  ,					encoding='utf-8') as f:
					f.write(json.dumps(self.encoder  ,					indent=2  ,					sort_keys=UpperCAmelCase_  ,					ensure_ascii=UpperCAmelCase_) + '\n')

			UpperCamelCase__				:					str					=					0
			with open(UpperCAmelCase_  ,					'w'  ,					encoding='utf-8') as writer:
					writer.write('#version: 0.2\n')
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,					key=lambda UpperCAmelCase_: kv[1]):
							if index != token_index:
									logger.warning(
									    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
									    ' Please check that the tokenizer is not corrupted!')
									UpperCamelCase__				:					List[Any]					=					token_index
							writer.write(' '.join(UpperCAmelCase_) + '\n')
							index += 1

			return vocab_file, merge_file

	def      __UpperCamelCase							(      self   :						Optional[int]  ,					UpperCAmelCase_   :						List[int]  ,					UpperCAmelCase_   :						Optional[List[int]] = None  ,					UpperCAmelCase_   :						bool = False):
			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=UpperCAmelCase_  ,					token_ids_a=UpperCAmelCase_  ,					already_has_special_tokens=UpperCAmelCase_)

			if token_ids_a is None:
					return [1] + ([0] * len(UpperCAmelCase_)) + [1]
			return [1] + ([0] * len(UpperCAmelCase_)) + [1, 1] + ([0] * len(UpperCAmelCase_)) + [1]

	def      __UpperCamelCase							(      self   :						List[str]  ,					UpperCAmelCase_   :						List[int]  ,					UpperCAmelCase_   :						Optional[List[int]] = None):
			UpperCamelCase__				:					Any					=					[self.sep_token_id]
			UpperCamelCase__				:					Optional[int]					=					[self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]

	def      __UpperCamelCase							(      self   :						str  ,					UpperCAmelCase_   :						Tuple  ,					UpperCAmelCase_   :						str=False  ,					**UpperCAmelCase_   :						Optional[Any]):
			UpperCamelCase__				:					Tuple					=					kwargs.pop('add_prefix_space'  ,					self.add_prefix_space)
			if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase_) > 0 and not text[0].isspace()):
					UpperCamelCase__				:					str					=					' ' + text
			return (text, kwargs)

	def      __UpperCamelCase							(      self   :						List[str]  ,					UpperCAmelCase_   :						List[int]  ,					UpperCAmelCase_   :						Optional[List[int]] = None):
			return token_ids_a + [self.eos_token_id]





	def      __UpperCamelCase							(      self   :						Dict  ,					UpperCAmelCase_   :						"Conversation"):
			UpperCamelCase__				:					List[str]					=					[]
			for is_user, text in conversation.iter_texts():
					if is_user:
							# We need to space prefix as it's being done within blenderbot
							inputs.append(' ' + text)
					else:
							# Generated responses should contain them already.
							inputs.append(UpperCAmelCase_)

			UpperCamelCase__				:					Optional[Any]					=					'  '.join(UpperCAmelCase_)
			UpperCamelCase__				:					int					=					self.encode(UpperCAmelCase_)
			if len(UpperCAmelCase_) > self.model_max_length:
					UpperCamelCase__				:					Optional[Any]					=					input_ids[-self.model_max_length :]
					logger.warning(F'Trimmed input from conversation as it was longer than {self.model_max_length} tokens.')
			return input_ids




