



"""simple docstring"""





import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__lowerCAmelCase       :					List[Any]      					=					datasets.utils.logging.get_logger(__name__)



@dataclass
class    _lowerCAmelCase    (				datasets.BuilderConfig		):





  """simple docstring"""






  _lowerCamelCase			  =       10_000
  _lowerCamelCase			  =       None
  _lowerCamelCase			  =       None




class    _lowerCAmelCase    (				datasets.ArrowBasedBuilder		):





  """simple docstring"""






  _lowerCamelCase			  =       ParquetConfig





  def    UpperCAmelCase__							(					self   ) ->						List[Any]:







        '''simple docstring'''

        return datasets.DatasetInfo(features=self.config.features   )





  def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						int:







        '''simple docstring'''

        if not self.config.data_files:
              raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'   )
        snake_case_   : str						=			dl_manager.download_and_extract(self.config.data_files   )
        if isinstance(_lowercase							,    (str, list, tuple)   ):
              snake_case_   : Union[str, Any]						=			data_files
              if isinstance(_lowercase							,    _lowercase   ):
                    snake_case_   : Any						=			[files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              snake_case_   : str						=			[dl_manager.iter_files(_lowercase   ) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,    gen_kwargs={"""files""": files}   )]
        snake_case_   : Union[str, Any]						=			[]
        for split_name, files in data_files.items():
              if isinstance(_lowercase							,    _lowercase   ):
                    snake_case_   : Optional[int]						=			[files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              snake_case_   : List[str]						=			[dl_manager.iter_files(_lowercase   ) for file in files]
              # Infer features is they are stoed in the arrow schema
              if self.info.features is None:
                    for file in itertools.chain.from_iterable(_lowercase   ):
                          with open(_lowercase							,    """rb"""   ) as f:
                                snake_case_   : List[str]						=			datasets.Features.from_arrow_schema(pq.read_schema(_lowercase   )   )
                          break
              splits.append(datasets.SplitGenerator(name=_lowercase							,    gen_kwargs={"""files""": files}   )   )
        return splits





  def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						pa.Table:







        '''simple docstring'''

        if self.info.features is not None:
              # more expensive cast to support nested features with keys in a different order
              # allows str <-> int/float or str to Audio for example
              snake_case_   : Dict						=			table_cast(_lowercase							,    self.info.features.arrow_schema   )
        return pa_table





  def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						str:







        '''simple docstring'''

        snake_case_   : Optional[int]						=			self.info.features.arrow_schema if self.info.features is not None else None
        if self.info.features is not None and self.config.columns is not None:
              if sorted(field.name for field in schema   ) != sorted(self.config.columns   ):
                    raise ValueError(
                        f'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''   )
        for file_idx, file in enumerate(itertools.chain.from_iterable(_lowercase   )   ):
              with open(_lowercase							,    """rb"""   ) as f:
                    snake_case_   : int						=			pq.ParquetFile(_lowercase   )
                    try:
                          for batch_idx, record_batch in enumerate(
                              parquet_file.iter_batches(batch_size=self.config.batch_size							,    columns=self.config.columns   )   ):
                                snake_case_   : Tuple						=			pa.Table.from_batches([record_batch]   )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield f'{file_idx}_{batch_idx}', self._cast_table(_lowercase   )
                    except ValueError as e:
                          logger.error(f'Failed to read file \'{file}\' with error {type(_lowercase   )}: {e}'   )
                          raise






"""simple docstring"""





from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)

