



"""simple docstring"""

import re
import string
from collections import Counter

import sacrebleu
import sacremoses
from packaging import version

import datasets


lowercase__			   =  "\n@inproceedings{xu-etal-2016-optimizing,\n    title = {Optimizing Statistical Machine Translation for Text Simplification},\n    authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},\n    journal = {Transactions of the Association for Computational Linguistics},\n    volume = {4},\n    year={2016},\n    url = {https://www.aclweb.org/anthology/Q16-1029},\n    pages = {401--415\n},\n@inproceedings{post-2018-call,\n    title = \"A Call for Clarity in Reporting {BLEU} Scores\",\n    author = \"Post, Matt\",\n    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",\n    month = oct,\n    year = \"2018\",\n    address = \"Belgium, Brussels\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://www.aclweb.org/anthology/W18-6319\",\n    pages = \"186--191\",\n}\n"

lowercase__			   =  "\\nWIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU\nIt can be used to evaluate the quality of machine-generated texts.\n"


lowercase__			   =  "\nCalculates sari score (between 0 and 100) given a list of source and predicted\nsentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score.\nArgs:\n    sources: list of source sentences where each sentence should be a string.\n    predictions: list of predicted sentences where each sentence should be a string.\n    references: list of lists of reference sentences where each sentence should be a string.\nReturns:\n    sari: sari score\n    sacrebleu: sacrebleu score\n    exact: exact score\n\nExamples:\n    >>> sources=[\"About 95 species are currently accepted .\"]\n    >>> predictions=[\"About 95 you now get in .\"]\n    >>> references=[[\"About 95 species are currently known .\"]]\n    >>> wiki_split = datasets.load_metric(\"wiki_split\")\n    >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references)\n    >>> print(results)\n    {'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0}\n"





def       __magic_name__    (   _lowerCamelCase				: Union[str, Any]					):

							def remove_articles(_lowerCamelCase				: List[str]					):
														__a		:		str									=       re.compile(r"""\b(a|an|the)\b"""  ,       re.UNICODE					)
														return re.sub(_lowerCamelCase  ,       """ """  ,       _lowerCamelCase					)

							def white_space_fix(_lowerCamelCase				: List[str]					):
														return " ".join(text.split()					)

							def remove_punc(_lowerCamelCase				: str					):
														__a		:		str									=       set(string.punctuation					)
														return "".join(ch for ch in text if ch not in exclude					)

							def lower(_lowerCamelCase				: Dict					):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(_lowerCamelCase					)					)					)					)





def       __magic_name__    (   _lowerCamelCase				: Any  ,       _lowerCamelCase				: int					):
							return int(normalize_answer(_lowerCamelCase					) == normalize_answer(_lowerCamelCase					)					)





def       __magic_name__    (   _lowerCamelCase				: Union[str, Any]  ,       _lowerCamelCase				: Tuple					):
							__a		:		str									=       [any(compute_exact(_lowerCamelCase  ,       _lowerCamelCase					) for ref in refs					) for pred, refs in zip(_lowerCamelCase  ,       _lowerCamelCase					)]
							return (sum(_lowerCamelCase					) / len(_lowerCamelCase					)) * 1_0_0





def       __magic_name__    (   _lowerCamelCase				: List[Any]  ,       _lowerCamelCase				: int  ,       _lowerCamelCase				: Dict  ,       _lowerCamelCase				: Optional[Any]					):
							__a		:		Tuple									=       [rgram for rgrams in rgramslist for rgram in rgrams]
							__a		:		Union[str, Any]									=       Counter(_lowerCamelCase					)

							__a		:		List[str]									=       Counter(_lowerCamelCase					)
							__a		:		str									=       Counter()
							for sgram, scount in sgramcounter.items():
														__a		:		Optional[int]									=       scount * numref

							__a		:		Optional[Any]									=       Counter(_lowerCamelCase					)
							__a		:		Optional[int]									=       Counter()
							for cgram, ccount in cgramcounter.items():
														__a		:		Dict									=       ccount * numref

							# KEEP
							__a		:		List[str]									=       sgramcounter_rep & cgramcounter_rep
							__a		:		Union[str, Any]									=       keepgramcounter_rep & rgramcounter
							__a		:		Union[str, Any]									=       sgramcounter_rep & rgramcounter

							__a		:		int									=       0
							__a		:		str									=       0
							for keepgram in keepgramcountergood_rep:
														keeptmpscorea += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram]
														# Fix an alleged bug [2] in the keep score computation.
														# keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram]
														keeptmpscorea += keepgramcountergood_rep[keepgram]
							# Define 0/0=1 instead of 0 to give higher scores for predictions that match
							#      a target exactly.
							__a		:		Optional[Any]									=       1
							__a		:		List[str]									=       1
							if len(_lowerCamelCase					) > 0:
														__a		:		List[Any]									=       keeptmpscorea / len(_lowerCamelCase					)
							if len(_lowerCamelCase					) > 0:
														# Fix an alleged bug [2] in the keep score computation.
														# keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep)
														__a		:		int									=       keeptmpscorea / sum(keepgramcounterall_rep.values()					)
							__a		:		List[Any]									=       0
							if keepscore_precision > 0 or keepscore_recall > 0:
														__a		:		Dict									=       2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall)

							# DELETION
							__a		:		Optional[Any]									=       sgramcounter_rep - cgramcounter_rep
							__a		:		Tuple									=       delgramcounter_rep - rgramcounter
							__a		:		int									=       sgramcounter_rep - rgramcounter
							__a		:		Any									=       0
							__a		:		Union[str, Any]									=       0
							for delgram in delgramcountergood_rep:
														deltmpscorea += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram]
														deltmpscorea += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram]
							# Define 0/0=1 instead of 0 to give higher scores for predictions that match
							# a target exactly.
							__a		:		Any									=       1
							if len(_lowerCamelCase					) > 0:
														__a		:		List[str]									=       deltmpscorea / len(_lowerCamelCase					)

							# ADDITION
							__a		:		Dict									=       set(_lowerCamelCase					) - set(_lowerCamelCase					)
							__a		:		Optional[int]									=       set(_lowerCamelCase					) & set(_lowerCamelCase					)
							__a		:		List[Any]									=       set(_lowerCamelCase					) - set(_lowerCamelCase					)

							__a		:		Tuple									=       0
							for addgram in addgramcountergood:
														addtmpscore += 1

							# Define 0/0=1 instead of 0 to give higher scores for predictions that match
							# a target exactly.
							__a		:		Union[str, Any]									=       1
							__a		:		Optional[Any]									=       1
							if len(_lowerCamelCase					) > 0:
														__a		:		Optional[Any]									=       addtmpscore / len(_lowerCamelCase					)
							if len(_lowerCamelCase					) > 0:
														__a		:		Tuple									=       addtmpscore / len(_lowerCamelCase					)
							__a		:		Tuple									=       0
							if addscore_precision > 0 or addscore_recall > 0:
														__a		:		str									=       2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall)

							return (keepscore, delscore_precision, addscore)





def       __magic_name__    (   _lowerCamelCase				: Optional[Any]  ,       _lowerCamelCase				: Optional[int]  ,       _lowerCamelCase				: Optional[int]					):
							__a		:		List[str]									=       len(_lowerCamelCase					)

							__a		:		str									=       ssent.split(""" """					)
							__a		:		str									=       csent.split(""" """					)
							__a		:		Optional[int]									=       []
							__a		:		Any									=       []
							__a		:		Dict									=       []
							__a		:		Tuple									=       []
							__a		:		str									=       []
							__a		:		Union[str, Any]									=       []

							__a		:		List[Any]									=       []
							__a		:		Dict									=       []
							__a		:		Union[str, Any]									=       []
							__a		:		Tuple									=       []
							for rsent in rsents:
														__a		:		str									=       rsent.split(""" """					)
														__a		:		Optional[int]									=       []
														__a		:		Optional[int]									=       []
														__a		:		Union[str, Any]									=       []
														ragramslist.append(_lowerCamelCase					)
														for i in range(0  ,       len(_lowerCamelCase					) - 1					):
																					if i < len(_lowerCamelCase					) - 1:
																												__a		:		str									=       ragrams[i] + """ """ + ragrams[i + 1]
																												ragrams.append(_lowerCamelCase					)
																					if i < len(_lowerCamelCase					) - 2:
																												__a		:		Dict									=       ragrams[i] + """ """ + ragrams[i + 1] + """ """ + ragrams[i + 2]
																												ragrams.append(_lowerCamelCase					)
																					if i < len(_lowerCamelCase					) - 3:
																												__a		:		List[str]									=       ragrams[i] + """ """ + ragrams[i + 1] + """ """ + ragrams[i + 2] + """ """ + ragrams[i + 3]
																												ragrams.append(_lowerCamelCase					)
														ragramslist.append(_lowerCamelCase					)
														ragramslist.append(_lowerCamelCase					)
														ragramslist.append(_lowerCamelCase					)

							for i in range(0  ,       len(_lowerCamelCase					) - 1					):
														if i < len(_lowerCamelCase					) - 1:
																					__a		:		List[Any]									=       sagrams[i] + """ """ + sagrams[i + 1]
																					sagrams.append(_lowerCamelCase					)
														if i < len(_lowerCamelCase					) - 2:
																					__a		:		Optional[Any]									=       sagrams[i] + """ """ + sagrams[i + 1] + """ """ + sagrams[i + 2]
																					sagrams.append(_lowerCamelCase					)
														if i < len(_lowerCamelCase					) - 3:
																					__a		:		Tuple									=       sagrams[i] + """ """ + sagrams[i + 1] + """ """ + sagrams[i + 2] + """ """ + sagrams[i + 3]
																					sagrams.append(_lowerCamelCase					)

							for i in range(0  ,       len(_lowerCamelCase					) - 1					):
														if i < len(_lowerCamelCase					) - 1:
																					__a		:		Dict									=       cagrams[i] + """ """ + cagrams[i + 1]
																					cagrams.append(_lowerCamelCase					)
														if i < len(_lowerCamelCase					) - 2:
																					__a		:		Optional[Any]									=       cagrams[i] + """ """ + cagrams[i + 1] + """ """ + cagrams[i + 2]
																					cagrams.append(_lowerCamelCase					)
														if i < len(_lowerCamelCase					) - 3:
																					__a		:		Optional[int]									=       cagrams[i] + """ """ + cagrams[i + 1] + """ """ + cagrams[i + 2] + """ """ + cagrams[i + 3]
																					cagrams.append(_lowerCamelCase					)

							((__a)							,     (__a)							,     (__a))		:		str									=       SARIngram(_lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase					)
							((__a)							,     (__a)							,     (__a))		:		Optional[Any]									=       SARIngram(_lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase					)
							((__a)							,     (__a)							,     (__a))		:		str									=       SARIngram(_lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase					)
							((__a)							,     (__a)							,     (__a))		:		str									=       SARIngram(_lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase					)
							__a		:		Optional[Any]									=       sum([keepascore, keepascore, keepascore, keepascore]					) / 4
							__a		:		List[str]									=       sum([delascore, delascore, delascore, delascore]					) / 4
							__a		:		Union[str, Any]									=       sum([addascore, addascore, addascore, addascore]					) / 4
							__a		:		int									=       (avgkeepscore + avgdelscore + avgaddscore) / 3
							return finalscore





def       __magic_name__    (   _lowerCamelCase				: Tuple  ,       _lowerCamelCase				: bool = True  ,       _lowerCamelCase				: str = "13a"  ,       _lowerCamelCase				: bool = True					):
							# Normalization is requried for the ASSET dataset (one of the primary
							# datasets in sentence simplification) to allow using space
							# to split the sentence. Even though Wiki-Auto and TURK datasets,
							# do not require normalization, we do it for consistency.
							# Code adapted from the EASSE library [1] written by the authors of the ASSET dataset.
							# [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7

							if lowercase:
														__a		:		Optional[Any]									=       sentence.lower()

							if tokenizer in ["13a", "intl"]:
														if version.parse(sacrebleu.__version__					).major >= 2:
																					__a		:		Tuple									=       sacrebleu.metrics.bleu._get_tokenizer(_lowerCamelCase					)()(_lowerCamelCase					)
														else:
																					__a		:		List[Any]									=       sacrebleu.TOKENIZERS[tokenizer]()(_lowerCamelCase					)
							elif tokenizer == "moses":
														__a		:		Tuple									=       sacremoses.MosesTokenizer().tokenize(_lowerCamelCase  ,       return_str=_lowerCamelCase  ,       escape=_lowerCamelCase					)
							elif tokenizer == "penn":
														__a		:		Tuple									=       sacremoses.MosesTokenizer().penn_tokenize(_lowerCamelCase  ,       return_str=_lowerCamelCase					)
							else:
														__a		:		str									=       sentence

							if not return_str:
														__a		:		Tuple									=       normalized_sent.split()

							return normalized_sent





def       __magic_name__    (   _lowerCamelCase				: Optional[int]  ,       _lowerCamelCase				: int  ,       _lowerCamelCase				: List[str]					):
							if not (len(_lowerCamelCase					) == len(_lowerCamelCase					) == len(_lowerCamelCase					)):
														raise ValueError("""Sources length must match predictions and references lengths."""					)
							__a		:		Optional[Any]									=       0
							for src, pred, refs in zip(_lowerCamelCase  ,       _lowerCamelCase  ,       _lowerCamelCase					):
														sari_score += SARIsent(normalize(_lowerCamelCase					)  ,       normalize(_lowerCamelCase					)  ,       [normalize(_lowerCamelCase					) for sent in refs]					)
							__a		:		Optional[int]									=       sari_score / len(_lowerCamelCase					)
							return 1_0_0 * sari_score





def       __magic_name__    (   _lowerCamelCase				: Optional[int]  ,       _lowerCamelCase				: Union[str, Any]  ,       _lowerCamelCase				: Dict="exp"  ,       _lowerCamelCase				: Optional[Any]=None  ,       _lowerCamelCase				: Optional[Any]=False  ,       _lowerCamelCase				: Union[str, Any]=False  ,       _lowerCamelCase				: List[str]=False  ,       ):
							__a		:		Any									=       len(references[0]					)
							if any(len(_lowerCamelCase					) != references_per_prediction for refs in references					):
														raise ValueError("""Sacrebleu requires the same number of references for each prediction"""					)
							__a		:		int									=       [[refs[i] for refs in references] for i in range(_lowerCamelCase					)]
							__a		:		Any									=       sacrebleu.corpus_bleu(
							    _lowerCamelCase  ,       _lowerCamelCase  ,       smooth_method=_lowerCamelCase  ,       smooth_value=_lowerCamelCase  ,       force=_lowerCamelCase  ,       lowercase=_lowerCamelCase  ,       use_effective_order=_lowerCamelCase  ,       )
							return output.score







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,					_KWARGS_DESCRIPTION      )
class SCREAMING_SNAKE_CASE__					(		datasets.Metric      ):



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													return datasets.MetricInfo(
													    description=_DESCRIPTION						,     citation=_CITATION						,     inputs_description=_KWARGS_DESCRIPTION						,     features=datasets.Features(
													        {
													            """predictions""": datasets.Value("""string"""						,     id="""sequence"""							),
													            """references""": datasets.Sequence(datasets.Value("""string"""						,     id="""sequence"""							)						,     id="""references"""							),
													        }							)						,     codebase_urls=[
													        """https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py""",
													        """https://github.com/cocoxu/simplification/blob/master/SARI.py""",
													        """https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py""",
													        """https://github.com/mjpost/sacreBLEU""",
													    ]						,     reference_urls=[
													        """https://www.aclweb.org/anthology/Q16-1029.pdf""",
													        """https://github.com/mjpost/sacreBLEU""",
													        """https://en.wikipedia.org/wiki/BLEU""",
													        """https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213""",
													    ]						,     )





						def     lowerCAmelCase__(self						,     _lowercase						,     _lowercase						,     _lowercase							):





													'''simple docstring'''

													__a		:		Any									=       {}
													result.update({"""sari""": compute_sari(sources=_lowercase						,     predictions=_lowercase						,     references=_lowercase							)}							)
													result.update({"""sacrebleu""": compute_sacrebleu(predictions=_lowercase						,     references=_lowercase							)}							)
													result.update({"""exact""": compute_em(predictions=_lowercase						,     references=_lowercase							)}							)
													return result






"""simple docstring"""

import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert_fast import BertTokenizerFast
from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer


lowercase__			   =  logging.get_logger(__name__)

lowercase__			   =  {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase__			   =  {
    "vocab_file": {
        "facebook/dpr-ctx_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt"
        ),
        "facebook/dpr-ctx_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "facebook/dpr-ctx_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json"
        ),
        "facebook/dpr-ctx_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json"
        ),
    },
}
lowercase__			   =  {
    "vocab_file": {
        "facebook/dpr-question_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt"
        ),
        "facebook/dpr-question_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "facebook/dpr-question_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json"
        ),
        "facebook/dpr-question_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json"
        ),
    },
}
lowercase__			   =  {
    "vocab_file": {
        "facebook/dpr-reader-single-nq-base": (
            "https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt"
        ),
        "facebook/dpr-reader-multiset-base": (
            "https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "facebook/dpr-reader-single-nq-base": (
            "https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json"
        ),
        "facebook/dpr-reader-multiset-base": (
            "https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json"
        ),
    },
}

lowercase__			   =  {
    "facebook/dpr-ctx_encoder-single-nq-base": 512,
    "facebook/dpr-ctx_encoder-multiset-base": 512,
}
lowercase__			   =  {
    "facebook/dpr-question_encoder-single-nq-base": 512,
    "facebook/dpr-question_encoder-multiset-base": 512,
}
lowercase__			   =  {
    "facebook/dpr-reader-single-nq-base": 512,
    "facebook/dpr-reader-multiset-base": 512,
}


lowercase__			   =  {
    "facebook/dpr-ctx_encoder-single-nq-base": {"do_lower_case": True},
    "facebook/dpr-ctx_encoder-multiset-base": {"do_lower_case": True},
}
lowercase__			   =  {
    "facebook/dpr-question_encoder-single-nq-base": {"do_lower_case": True},
    "facebook/dpr-question_encoder-multiset-base": {"do_lower_case": True},
}
lowercase__			   =  {
    "facebook/dpr-reader-single-nq-base": {"do_lower_case": True},
    "facebook/dpr-reader-multiset-base": {"do_lower_case": True},
}




class SCREAMING_SNAKE_CASE__					(		__snake_case      ):
						_lowerCAmelCase							=					VOCAB_FILES_NAMES
						_lowerCAmelCase							=					CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
						_lowerCAmelCase							=					CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowerCAmelCase							=					CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
						_lowerCAmelCase							=					DPRContextEncoderTokenizer







class SCREAMING_SNAKE_CASE__					(		__snake_case      ):
						_lowerCAmelCase							=					VOCAB_FILES_NAMES
						_lowerCAmelCase							=					QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
						_lowerCAmelCase							=					QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowerCAmelCase							=					QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
						_lowerCAmelCase							=					DPRQuestionEncoderTokenizer







lowercase__			   =  collections.namedtuple(
    "DPRSpanPrediction", ["span_score", "relevance_score", "doc_id", "start_index", "end_index", "text"]
)

lowercase__			   =  collections.namedtuple("DPRReaderOutput", ["start_logits", "end_logits", "relevance_logits"])


lowercase__			   =  R"\n    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.\n    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),\n    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`\n    with the format:\n\n    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>\n\n    Args:\n        questions (`str` or `List[str]`):\n            The questions to be encoded. You can specify one question for many passages. In this case, the question\n            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in\n            `titles` or `texts`.\n        titles (`str` or `List[str]`):\n            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.\n        texts (`str` or `List[str]`):\n            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.\n        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):\n            Activates and controls padding. Accepts the following values:\n\n            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence\n              if provided).\n            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided.\n            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different\n              lengths).\n        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):\n            Activates and controls truncation. Accepts the following values:\n\n            - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to\n              the maximum acceptable input length for the model if that argument is not provided. This will truncate\n              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch\n              of pairs) is provided.\n            - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the first\n              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the\n              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths\n              greater than the model maximum admissible input size).\n        max_length (`int`, *optional*):\n                Controls the maximum length to use by one of the truncation/padding parameters.\n\n                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length\n                is required by one of the truncation/padding parameters. If the model has no specific maximum input\n                length (like XLNet) truncation/padding to a maximum length will be deactivated.\n        return_tensors (`str` or [`~utils.TensorType`], *optional*):\n                If set, will return tensors instead of list of python integers. Acceptable values are:\n\n                - `'tf'`: Return TensorFlow `tf.constant` objects.\n                - `'pt'`: Return PyTorch `torch.Tensor` objects.\n                - `'np'`: Return Numpy `np.ndarray` objects.\n        return_attention_mask (`bool`, *optional*):\n            Whether or not to return the attention mask. If not set, will return the attention mask according to the\n            specific tokenizer's default, defined by the `return_outputs` attribute.\n\n            [What are attention masks?](../glossary#attention-mask)\n\n    Return:\n        `Dict[str, List[List[int]]]`: A dictionary with the following keys:\n\n        - `input_ids`: List of token ids to be fed to a model.\n        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.\n    "




@add_start_docstrings(__snake_case      )
class SCREAMING_SNAKE_CASE__					:



						def __call__(self						,     _lowercase						,     _lowercase = None						,     _lowercase = None						,     _lowercase = False						,     _lowercase = False						,     _lowercase = None						,     _lowercase = None						,     _lowercase = None						,     **_lowercase						,     ):





													'''simple docstring'''

													if titles is None and texts is None:
																				return super().__call__(
																				    _lowercase						,     padding=_lowercase						,     truncation=_lowercase						,     max_length=_lowercase						,     return_tensors=_lowercase						,     return_attention_mask=_lowercase						,     **_lowercase						,     )
													elif titles is None or texts is None:
																				__a		:		str									=       titles if texts is None else texts
																				return super().__call__(
																				    _lowercase						,     _lowercase						,     padding=_lowercase						,     truncation=_lowercase						,     max_length=_lowercase						,     return_tensors=_lowercase						,     return_attention_mask=_lowercase						,     **_lowercase						,     )
													__a		:		str									=       titles if not isinstance(_lowercase						,     _lowercase							) else [titles]
													__a		:		Optional[Any]									=       texts if not isinstance(_lowercase						,     _lowercase							) else [texts]
													__a		:		Tuple									=       len(_lowercase							)
													__a		:		Dict									=       questions if not isinstance(_lowercase						,     _lowercase							) else [questions] * n_passages
													assert len(_lowercase							) == len(
													    _lowercase							), F'''There should be as many titles than texts but got {len(_lowercase							)} titles and {len(_lowercase							)} texts.'''
													__a		:		Optional[Any]									=       super().__call__(_lowercase						,     _lowercase						,     padding=_lowercase						,     truncation=_lowercase							)["""input_ids"""]
													__a		:		str									=       super().__call__(_lowercase						,     add_special_tokens=_lowercase						,     padding=_lowercase						,     truncation=_lowercase							)["""input_ids"""]
													__a		:		Union[str, Any]									=       {
													    """input_ids""": [
													        (encoded_question_and_title + encoded_text)[:max_length]
													        if max_length is not None and truncation
													        else encoded_question_and_title + encoded_text
													        for encoded_question_and_title, encoded_text in zip(_lowercase						,     _lowercase							)
													    ]
													}
													if return_attention_mask is not False:
																				__a		:		Optional[int]									=       []
																				for input_ids in encoded_inputs["input_ids"]:
																											attention_mask.append([int(input_id != self.pad_token_id							) for input_id in input_ids]							)
																				__a		:		str									=       attention_mask
													return self.pad(_lowercase						,     padding=_lowercase						,     max_length=_lowercase						,     return_tensors=_lowercase							)



						def     lowerCAmelCase__(self						,     _lowercase						,     _lowercase						,     _lowercase = 16						,     _lowercase = 64						,     _lowercase = 4						,     ):





													'''simple docstring'''

													__a		:		Union[str, Any]									=       reader_input["""input_ids"""]
													__a							,     __a							,     __a		:		Optional[int]									=       reader_output[:3]
													__a		:		int									=       len(_lowercase							)
													__a		:		Any									=       sorted(range(_lowercase							)						,     reverse=_lowercase						,     key=relevance_logits.__getitem__							)
													__a		:		List[DPRReaderOutput]									=       []
													for doc_id in sorted_docs:
																				__a		:		Optional[int]									=       list(input_ids[doc_id]							)
																				# assuming question & title information is at the beginning of the sequence
																				__a		:		Dict									=       sequence_ids.index(self.sep_token_id						,     2							) + 1  # second sep id
																				if sequence_ids[-1] == self.pad_token_id:
																											__a		:		int									=       sequence_ids.index(self.pad_token_id							)
																				else:
																											__a		:		Optional[Any]									=       len(_lowercase							)

																				__a		:		List[Any]									=       self._get_best_spans(
																				    start_logits=start_logits[doc_id][passage_offset:sequence_len]						,     end_logits=end_logits[doc_id][passage_offset:sequence_len]						,     max_answer_length=_lowercase						,     top_spans=_lowercase						,     )
																				for start_index, end_index in best_spans:
																											start_index += passage_offset
																											end_index += passage_offset
																											nbest_spans_predictions.append(
																											    DPRSpanPrediction(
																											        span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]						,     relevance_score=relevance_logits[doc_id]						,     doc_id=_lowercase						,     start_index=_lowercase						,     end_index=_lowercase						,     text=self.decode(sequence_ids[start_index : end_index + 1]							)						,     )							)
																				if len(_lowercase							) >= num_spans:
																											break
													return nbest_spans_predictions[:num_spans]





						def     lowerCAmelCase__(self						,     _lowercase						,     _lowercase						,     _lowercase						,     _lowercase						,     ):





													'''simple docstring'''

													__a		:		Tuple									=       []
													for start_index, start_score in enumerate(_lowercase							):
																				for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]							):
																											scores.append(((start_index, start_index + answer_length), start_score + end_score)							)
													__a		:		str									=       sorted(_lowercase						,     key=lambda _lowercase							: x[1]						,     reverse=_lowercase							)
													__a		:		Union[str, Any]									=       []
													for (start_index, end_index), score in scores:
																				assert start_index <= end_index, F'''Wrong span indices: [{start_index}:{end_index}]'''
																				__a		:		List[str]									=       end_index - start_index + 1
																				assert length <= max_answer_length, F'''Span is too long: {length} > {max_answer_length}'''
																				if any(
																				    start_index <= prev_start_index <= prev_end_index <= end_index
																				    or prev_start_index <= start_index <= end_index <= prev_end_index
																				    for (prev_start_index, prev_end_index) in chosen_span_intervals							):
																											continue
																				chosen_span_intervals.append((start_index, end_index)							)

																				if len(_lowercase							) == top_spans:
																											break
													return chosen_span_intervals







@add_end_docstrings(__snake_case      )
class SCREAMING_SNAKE_CASE__					(		__snake_case     ,					__snake_case      ):
						_lowerCAmelCase							=					VOCAB_FILES_NAMES
						_lowerCAmelCase							=					READER_PRETRAINED_VOCAB_FILES_MAP
						_lowerCAmelCase							=					READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowerCAmelCase							=					READER_PRETRAINED_INIT_CONFIGURATION
						_lowerCAmelCase							=					["input_ids", "attention_mask"]
						_lowerCAmelCase							=					DPRReaderTokenizer

