





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase 				=     logging.get_logger(__name__)


_lowerCAmelCase 				=     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
_lowerCAmelCase 				=     {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

_lowerCAmelCase 				=     {
    """facebook/bart-base""": 1_024,
    """facebook/bart-large""": 1_024,
    """facebook/bart-large-mnli""": 1_024,
    """facebook/bart-large-cnn""": 1_024,
    """facebook/bart-large-xsum""": 1_024,
    """yjernite/bart_eli5""": 1_024,
}



@lru_cache()
def     _lowerCAmelCase		(  ):
			'''simple docstring'''


			A_     :				List[str]			      =	(
			    list(range(ord("""!"""     )  ,ord("""~"""     ) + 1     )     ) + list(range(ord("""¡"""     )  ,ord("""¬"""     ) + 1     )     ) + list(range(ord("""®"""     )  ,ord("""ÿ"""     ) + 1     )     )
			)
			A_     :				List[str]			      =	bs[:]
			A_     :				List[Any]			      =	0
			for b in range(2**8     ):
						if b not in bs:
									bs.append(_UpperCamelCase     )
									cs.append(2**8 + n     )
									n += 1
			A_     :				Optional[Any]			      =	[chr(_UpperCamelCase     ) for n in cs]
			return dict(zip(_UpperCamelCase  ,_UpperCamelCase     )     )



def     _lowerCAmelCase		(  _lowerCAmelCase     ):
			'''simple docstring'''


			A_     :				List[str]			      =	set()
			A_     :				Any			      =	word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)     )
						A_     :				Optional[int]			      =	char
			return pairs







class    _UpperCAmelCase							(		_lowerCamelCase     ):
						a						       =							VOCAB_FILES_NAMES
						a						       =							PRETRAINED_VOCAB_FILES_MAP
						a						       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a						       =							["""input_ids""", """attention_mask"""]
						def __init__(		self      ,       a__      ,       a__      ,       a__="replace"      ,       a__="<s>"      ,       a__="</s>"      ,       a__="</s>"      ,       a__="<s>"      ,       a__="<unk>"      ,       a__="<pad>"      ,       a__="<mask>"      ,       a__=False      ,       **a__      ,       ):
									A_     :				List[str]			      =	AddedToken(a__      ,       lstrip=a__      ,       rstrip=a__       ) if isinstance(a__      ,       a__       ) else bos_token
									A_     :				List[Any]			      =	AddedToken(a__      ,       lstrip=a__      ,       rstrip=a__       ) if isinstance(a__      ,       a__       ) else eos_token
									A_     :				str			      =	AddedToken(a__      ,       lstrip=a__      ,       rstrip=a__       ) if isinstance(a__      ,       a__       ) else sep_token
									A_     :				Tuple			      =	AddedToken(a__      ,       lstrip=a__      ,       rstrip=a__       ) if isinstance(a__      ,       a__       ) else cls_token
									A_     :				Dict			      =	AddedToken(a__      ,       lstrip=a__      ,       rstrip=a__       ) if isinstance(a__      ,       a__       ) else unk_token
									A_     :				List[Any]			      =	AddedToken(a__      ,       lstrip=a__      ,       rstrip=a__       ) if isinstance(a__      ,       a__       ) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									A_     :				Any			      =	AddedToken(a__      ,       lstrip=a__      ,       rstrip=a__       ) if isinstance(a__      ,       a__       ) else mask_token

									super().__init__(
									    errors=a__      ,       bos_token=a__      ,       eos_token=a__      ,       unk_token=a__      ,       sep_token=a__      ,       cls_token=a__      ,       pad_token=a__      ,       mask_token=a__      ,       add_prefix_space=a__      ,       **a__      ,       )

									with open(a__      ,       encoding="""utf-8"""       ) as vocab_handle:
												A_     :				Any			      =	json.load(a__       )
									A_     :				List[Any]			      =	{v: k for k, v in self.encoder.items()}
									A_     :				Any			      =	errors  # how to handle errors in decoding
									A_     :				List[str]			      =	bytes_to_unicode()
									A_     :				str			      =	{v: k for k, v in self.byte_encoder.items()}
									with open(a__      ,       encoding="""utf-8"""       ) as merges_handle:
												A_     :				str			      =	merges_handle.read().split("""\n"""       )[1:-1]
									A_     :				Tuple			      =	[tuple(merge.split()       ) for merge in bpe_merges]
									A_     :				Optional[int]			      =	dict(zip(a__      ,       range(len(a__       )       )       )       )
									A_     :				Dict			      =	{}
									A_     :				List[Any]			      =	add_prefix_space

									# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
									A_     :				Optional[int]			      =	re.compile(R"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )
						@property
						def 		_lowerCamelCase      (		self       ):
									return len(self.encoder       )
						def 		_lowerCamelCase      (		self       ):
									return dict(self.encoder      ,       **self.added_tokens_encoder       )
						def 		_lowerCamelCase      (		self      ,       a__       ):

									if token in self.cache:
												return self.cache[token]
									A_     :				Union[str, Any]			      =	tuple(a__       )
									A_     :				str			      =	get_pairs(a__       )

									if not pairs:
												return token

									while True:
												A_     :				List[str]			      =	min(a__      ,       key=lambda a__       : self.bpe_ranks.get(a__      ,       float("""inf"""       )       )       )
												if bigram not in self.bpe_ranks:
															break
												A_      ,				A_     :				Optional[Any]			      =	bigram
												A_     :				Tuple			      =	[]
												A_     :				Tuple			      =	0
												while i < len(a__       ):
															try:
																		A_     :				Dict			      =	word.index(a__      ,       a__       )
															except ValueError:
																		new_word.extend(word[i:]       )
																		break
															else:
																		new_word.extend(word[i:j]       )
																		A_     :				int			      =	j

															if word[i] == first and i < len(a__       ) - 1 and word[i + 1] == second:
																		new_word.append(first + second       )
																		i += 2
															else:
																		new_word.append(word[i]       )
																		i += 1
												A_     :				List[Any]			      =	tuple(a__       )
												A_     :				Tuple			      =	new_word
												if len(a__       ) == 1:
															break
												else:
															A_     :				Optional[Any]			      =	get_pairs(a__       )
									A_     :				str			      =	""" """.join(a__       )
									A_     :				Optional[Any]			      =	word
									return word
						def 		_lowerCamelCase      (		self      ,       a__       ):
									A_     :				Optional[Any]			      =	[]
									for token in re.findall(self.pat      ,       a__       ):
												A_     :				Optional[Any]			      =	"""""".join(
												    self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a__       ).split(""" """       )       )
									return bpe_tokens
						def 		_lowerCamelCase      (		self      ,       a__       ):
									return self.encoder.get(a__      ,       self.encoder.get(self.unk_token       )       )
						def 		_lowerCamelCase      (		self      ,       a__       ):
									return self.decoder.get(a__       )
						def 		_lowerCamelCase      (		self      ,       a__       ):
									A_     :				int			      =	"""""".join(a__       )
									A_     :				List[str]			      =	bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""      ,       errors=self.errors       )
									return text
						def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):

									if not os.path.isdir(a__       ):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
												return
									A_     :				List[Any]			      =	os.path.join(
									    a__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
									A_     :				Tuple			      =	os.path.join(
									    a__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

									with open(a__      ,       """w"""      ,       encoding="""utf-8"""       ) as f:
												f.write(json.dumps(self.encoder      ,       indent=2      ,       sort_keys=a__      ,       ensure_ascii=a__       ) + """\n"""       )

									A_     :				Dict			      =	0
									with open(a__      ,       """w"""      ,       encoding="""utf-8"""       ) as writer:
												writer.write("""#version: 0.2\n"""       )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,       key=lambda a__       : kv[1]       ):
															if index != token_index:
																		logger.warning(
																		    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																		    """ Please check that the tokenizer is not corrupted!"""       )
																		A_     :				Any			      =	token_index
															writer.write(""" """.join(a__       ) + """\n"""       )
															index += 1

									return vocab_file, merge_file
						def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):

									if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									A_     :				List[str]			      =	[self.cls_token_id]
									A_     :				Optional[Any]			      =	[self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep
						def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None      ,       a__ = False       ):

									if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=a__      ,       token_ids_a=a__      ,       already_has_special_tokens=a__       )

									if token_ids_a is None:
												return [1] + ([0] * len(a__       )) + [1]
									return [1] + ([0] * len(a__       )) + [1, 1] + ([0] * len(a__       )) + [1]
						def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):
									A_     :				Optional[Any]			      =	[self.sep_token_id]
									A_     :				List[Any]			      =	[self.cls_token_id]

									if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]






						def 		_lowerCamelCase      (		self      ,       a__      ,       a__=False      ,       **a__       ):
									A_     :				List[str]			      =	kwargs.pop("""add_prefix_space"""      ,       self.add_prefix_space       )
									if (is_split_into_words or add_prefix_space) and (len(a__       ) > 0 and not text[0].isspace()):
												A_     :				int			      =	""" """ + text
									return (text, kwargs)













from . import __version__

# Backward compatibility imports, to make sure all those objects can be found in file_utils
from .utils import (
    CLOUDFRONT_DISTRIB_PREFIX,
    CONFIG_NAME,
    DISABLE_TELEMETRY,
    DUMMY_INPUTS,
    DUMMY_MASK,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    FEATURE_EXTRACTOR_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    MODEL_CARD_NAME,
    MULTIPLE_CHOICE_DUMMY_INPUTS,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    SENTENCEPIECE_UNDERLINE,
    SPIECE_UNDERLINE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TORCH_FX_REQUIRED_VERSION,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
    ContextManagers,
    DummyObject,
    EntryNotFoundError,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    TensorType,
    _LazyModule,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    cached_property,
    copy_func,
    default_cache_path,
    define_sagemaker_information,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    get_torch_version,
    has_file,
    http_user_agent,
    is_apex_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_datasets_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_librosa_available,
    is_offline_mode,
    is_onnx_available,
    is_pandas_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_tensor,
    is_tensorflow_probability_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    replace_return_docstrings,
    requires_backends,
    to_numpy,
    to_py_obj,
    torch_only_method,
)






