





import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader



def     _lowerCAmelCase		(  _lowerCAmelCase     ):
   '''simple docstring'''


   A_     :				str			      =	np.inf

   def set_batch_size(_lowerCAmelCase     ) -> None:
      nonlocal batch_size
      if isinstance(_lowerCAmelCase  ,_lowerCAmelCase     ):
         A_     :				List[str]			      =	min(_lowerCAmelCase  ,config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS     )
      elif isinstance(_lowerCAmelCase  ,_lowerCAmelCase     ):
         A_     :				Optional[int]			      =	min(_lowerCAmelCase  ,config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS     )
      elif isinstance(_lowerCAmelCase  ,_lowerCAmelCase     ) and feature.dtype == "binary":
         A_     :				Tuple			      =	min(_lowerCAmelCase  ,config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS     )

   _visit(_lowerCAmelCase  ,_lowerCAmelCase     )

   return None if batch_size is np.inf else batch_size



class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__      ,       a__ = None      ,       a__ = None      ,       a__ = None      ,       a__ = False      ,       a__ = False      ,       a__ = None      ,       **a__      ,       ):
         super().__init__(
             a__      ,       split=a__      ,       features=a__      ,       cache_dir=a__      ,       keep_in_memory=a__      ,       streaming=a__      ,       num_proc=a__      ,       **a__      ,       )
         A_     :				List[str]			      =	path_or_paths if isinstance(a__      ,       a__       ) else {self.split: path_or_paths}
         A_     :				Dict			      =	_PACKAGED_DATASETS_MODULES["""parquet"""][1]
         A_     :				Tuple			      =	Parquet(
             cache_dir=a__      ,       data_files=a__      ,       features=a__      ,       hash=a__      ,       **a__      ,       )






      def 		_lowerCamelCase      (		self       ):
         # Build iterable dataset
         if self.streaming:
            A_     :				Dict			      =	self.builder.as_streaming_dataset(split=self.split       )
         # Build regular (map-style) dataset
         else:
            A_     :				List[Any]			      =	None
            A_     :				Tuple			      =	None
            A_     :				Dict			      =	None
            A_     :				List[Any]			      =	None

            self.builder.download_and_prepare(
                download_config=a__      ,       download_mode=a__      ,       verification_mode=a__      ,       base_path=a__      ,       num_proc=self.num_proc      ,       )
            A_     :				Dict			      =	self.builder.as_dataset(
                split=self.split      ,       verification_mode=a__      ,       in_memory=self.keep_in_memory       )
         return dataset






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__      ,       a__ = None      ,       **a__      ,       ):
         A_     :				Tuple			      =	dataset
         A_     :				Optional[Any]			      =	path_or_buf
         A_     :				Any			      =	batch_size or get_writer_batch_size(dataset.features       )
         A_     :				Dict			      =	parquet_writer_kwargs
      def 		_lowerCamelCase      (		self       ):
         A_     :				Union[str, Any]			      =	self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

         if isinstance(self.path_or_buf      ,       (str, bytes, os.PathLike)       ):
            with open(self.path_or_buf      ,       """wb+"""       ) as buffer:
               A_     :				Optional[Any]			      =	self._write(file_obj=a__      ,       batch_size=a__      ,       **self.parquet_writer_kwargs       )
         else:
            A_     :				Optional[Any]			      =	self._write(file_obj=self.path_or_buf      ,       batch_size=a__      ,       **self.parquet_writer_kwargs       )
         return written






      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       **a__       ):
         A_     :				Tuple			      =	0
         A_     :				Dict			      =	parquet_writer_kwargs.pop("""path_or_buf"""      ,       a__       )
         A_     :				Optional[Any]			      =	self.dataset.features.arrow_schema

         A_     :				Dict			      =	pq.ParquetWriter(a__      ,       schema=a__      ,       **a__       )

         for offset in logging.tqdm(
             range(0      ,       len(self.dataset       )      ,       a__       )      ,       unit="""ba"""      ,       disable=not logging.is_progress_bar_enabled()      ,       desc="""Creating parquet from Arrow format"""      ,       ):
            A_     :				List[Any]			      =	query_table(
                table=self.dataset._data      ,       key=slice(a__      ,       offset + batch_size       )      ,       indices=self.dataset._indices if self.dataset._indices is not None else None      ,       )
            writer.write_table(a__       )
            written += batch.nbytes
         writer.close()
         return written













import datasets

from .evaluate import evaluate


_lowerCAmelCase 				=     """\
@inproceedings{Rajpurkar2016SQuAD10,
  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},
  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
  booktitle={EMNLP},
  year={2016}
}
"""

_lowerCAmelCase 				=     """
This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).

Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by
crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,
from the corresponding reading passage, or the question might be unanswerable.
"""

_lowerCAmelCase 				=     """
Computes SQuAD scores (F1 and EM).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair as given in the references (see below)
        - 'prediction_text': the text of the answer
    references: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair (see above),
        - 'answers': a Dict in the SQuAD dataset format
            {
                'text': list of possible texts for the answer, as a list of strings
                'answer_start': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    'exact_match': Exact match (the normalized answer exactly match the gold answer)
    'f1': The F-score of predicted tokens versus the gold answer
Examples:

    >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}]
    >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}]
    >>> squad_metric = datasets.load_metric(\"squad\")
    >>> results = squad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 100.0, 'f1': 100.0}
"""

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,       _KWARGS_DESCRIPTION     )
class    _UpperCAmelCase							(		datasets.Metric     ):
      def 		_lowerCamelCase      (		self       ):
         return datasets.MetricInfo(
             description=_DESCRIPTION      ,       citation=_CITATION      ,       inputs_description=_KWARGS_DESCRIPTION      ,       features=datasets.Features(
                 {
                     """predictions""": {"""id""": datasets.Value("""string"""       ), """prediction_text""": datasets.Value("""string"""       )},
                     """references""": {
                         """id""": datasets.Value("""string"""       ),
                         """answers""": datasets.features.Sequence(
                             {
                                 """text""": datasets.Value("""string"""       ),
                                 """answer_start""": datasets.Value("""int32"""       ),
                             }       ),
                     },
                 }       )      ,       codebase_urls=["""https://rajpurkar.github.io/SQuAD-explorer/"""]      ,       reference_urls=["""https://rajpurkar.github.io/SQuAD-explorer/"""]      ,       )






      def 		_lowerCamelCase      (		self      ,       a__      ,       a__       ):
         A_     :				Union[str, Any]			      =	{prediction["""id"""]: prediction["""prediction_text"""] for prediction in predictions}
         A_     :				Optional[Any]			      =	[
             {
                 """paragraphs""": [
                     {
                         """qas""": [
                             {
                                 """answers""": [{"""text""": answer_text} for answer_text in ref["""answers"""]["""text"""]],
                                 """id""": ref["""id"""],
                             }
                             for ref in references
                         ]
                     }
                 ]
             }
         ]
         A_     :				Union[str, Any]			      =	evaluate(dataset=a__      ,       predictions=a__       )
         return score






