






import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


a_		:						Optional[int]             =    datasets.utils.logging.get_logger(__name__)

a_		:						Union[str, Any]             =    ['names', 'prefix']
a_		:						Union[str, Any]             =    ['warn_bad_lines', 'error_bad_lines', 'mangle_dupe_cols']
a_		:						List[Any]             =    ['encoding_errors', 'on_bad_lines']
a_		:						Tuple             =    ['date_format']

@dataclass
class 		_snake_case  (  datasets.BuilderConfig					):
			_lowercase		:    str			     =							","
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    Optional[Union[int, List[int], str]]			     =							"infer"
			_lowercase		:    Optional[List[str]]			     =							None
			_lowercase		:    Optional[List[str]]			     =							None
			_lowercase		:    Optional[Union[int, str, List[int], List[str]]]			     =							None
			_lowercase		:    Optional[Union[List[int], List[str]]]			     =							None
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    bool			     =							True
			_lowercase		:    Optional[Literal["c", "python", "pyarrow"]]			     =							None
			_lowercase		:    Dict[Union[int, str], Callable[[Any], Any]]			     =							None
			_lowercase		:    Optional[list]			     =							None
			_lowercase		:    Optional[list]			     =							None
			_lowercase		:    bool			     =							False
			_lowercase		:    Optional[Union[int, List[int]]]			     =							None
			_lowercase		:    Optional[int]			     =							None
			_lowercase		:    Optional[Union[str, List[str]]]			     =							None
			_lowercase		:    bool			     =							True
			_lowercase		:    bool			     =							True
			_lowercase		:    bool			     =							False
			_lowercase		:    bool			     =							True
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    str			     =							"."
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    str			     =							'"'
			_lowercase		:    int			     =							0
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    bool			     =							True
			_lowercase		:    bool			     =							True
			_lowercase		:    int			     =							0
			_lowercase		:    bool			     =							True
			_lowercase		:    bool			     =							False
			_lowercase		:    Optional[str]			     =							None
			_lowercase		:    int			     =							1_00_00
			_lowercase		:    Optional[datasets.Features]			     =							None
			_lowercase		:    Optional[str]			     =							"strict"
			_lowercase		:    Literal["error", "warn", "skip"]			     =							"error"
			_lowercase		:    Optional[str]			     =							None






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Dict:
						if self.delimiter is not None:
									SCREAMING_SNAKE_CASE						= self.delimiter
						if self.column_names is not None:
									SCREAMING_SNAKE_CASE						= self.column_names







			@property
			def    SCREAMING_SNAKE_CASE__    (       self)      ->       int:
						SCREAMING_SNAKE_CASE						= {
						    'sep': self.sep,
						    'header': self.header,
						    'names': self.names,
						    'index_col': self.index_col,
						    'usecols': self.usecols,
						    'prefix': self.prefix,
						    'mangle_dupe_cols': self.mangle_dupe_cols,
						    'engine': self.engine,
						    'converters': self.converters,
						    'true_values': self.true_values,
						    'false_values': self.false_values,
						    'skipinitialspace': self.skipinitialspace,
						    'skiprows': self.skiprows,
						    'nrows': self.nrows,
						    'na_values': self.na_values,
						    'keep_default_na': self.keep_default_na,
						    'na_filter': self.na_filter,
						    'verbose': self.verbose,
						    'skip_blank_lines': self.skip_blank_lines,
						    'thousands': self.thousands,
						    'decimal': self.decimal,
						    'lineterminator': self.lineterminator,
						    'quotechar': self.quotechar,
						    'quoting': self.quoting,
						    'escapechar': self.escapechar,
						    'comment': self.comment,
						    'encoding': self.encoding,
						    'dialect': self.dialect,
						    'error_bad_lines': self.error_bad_lines,
						    'warn_bad_lines': self.warn_bad_lines,
						    'skipfooter': self.skipfooter,
						    'doublequote': self.doublequote,
						    'memory_map': self.memory_map,
						    'float_precision': self.float_precision,
						    'chunksize': self.chunksize,
						    'encoding_errors': self.encoding_errors,
						    'on_bad_lines': self.on_bad_lines,
						    'date_format': self.date_format,
						}

						# some kwargs must not be passed if they don't have a default value
						# some others are deprecated and we can also not pass them if they are the default value
						for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
									if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()		,					a):
												del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
						if not (datasets.config.PANDAS_VERSION.major >= 2):
									for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
												del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
						if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
									for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
												del pd_read_csv_kwargs[pd_read_csv_parameter]

						return pd_read_csv_kwargs



class 		_snake_case  (  datasets.ArrowBasedBuilder					):
			_lowercase		:    Dict			     =							CsvConfig






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Union[str, Any]:
						return datasets.DatasetInfo(features=self.config.features)






			def    SCREAMING_SNAKE_CASE__    (       self		,					a)      ->       str:
						if not self.config.data_files:
									raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}''')
						SCREAMING_SNAKE_CASE						= dl_manager.download_and_extract(self.config.data_files)
						if isinstance(a		,					(str, list, tuple)):
									SCREAMING_SNAKE_CASE						= data_files
									if isinstance(a		,					a):
												SCREAMING_SNAKE_CASE						= [files]
									SCREAMING_SNAKE_CASE						= [dl_manager.iter_files(a) for file in files]
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,					gen_kwargs={'files': files})]
						SCREAMING_SNAKE_CASE						= []
						for split_name, files in data_files.items():
									if isinstance(a		,					a):
												SCREAMING_SNAKE_CASE						= [files]
									SCREAMING_SNAKE_CASE						= [dl_manager.iter_files(a) for file in files]
									splits.append(datasets.SplitGenerator(name=a		,					gen_kwargs={'files': files}))
						return splits






			def    SCREAMING_SNAKE_CASE__    (       self		,					a)      ->       pa.Table:
						if self.config.features is not None:
									SCREAMING_SNAKE_CASE						= self.config.features.arrow_schema
									if all(not require_storage_cast(a) for feature in self.config.features.values()):
												# cheaper cast
												SCREAMING_SNAKE_CASE						= pa.Table.from_arrays([pa_table[field.name] for field in schema]		,					schema=a)
									else:
												# more expensive cast; allows str <-> int/float or str to Audio for example
												SCREAMING_SNAKE_CASE						= table_cast(a		,					a)
						return pa_table







			def    SCREAMING_SNAKE_CASE__    (       self		,					a)      ->       Dict:
						SCREAMING_SNAKE_CASE						= self.config.features.arrow_schema if self.config.features else None
						# dtype allows reading an int column as str
						SCREAMING_SNAKE_CASE						= (
						    {
						        name: dtype.to_pandas_dtype() if not require_storage_cast(a) else object
						        for name, dtype, feature in zip(schema.names		,					schema.types		,					self.config.features.values())
						    }
						    if schema is not None
						    else None
						)
						for file_idx, file in enumerate(itertools.chain.from_iterable(a)):
									SCREAMING_SNAKE_CASE						= pd.read_csv(a		,					iterator=a		,					dtype=a		,					**self.config.pd_read_csv_kwargs)
									try:
												for batch_idx, df in enumerate(a):
															SCREAMING_SNAKE_CASE						= pa.Table.from_pandas(a)
															# Uncomment for debugging (will print the Arrow table size and elements)
															# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
															# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
															yield (file_idx, batch_idx), self._cast_table(a)
									except ValueError as e:
												logger.error(f'''Failed to read file \'{file}\' with error {type(a)}: {e}''')
												raise












import copy

from ...configuration_utils import PretrainedConfig
from ...utils import add_start_docstrings


a_		:						List[str]             =    R'\n    [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and\n    can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.\n\n    Args:\n        title_sep (`str`, *optional*, defaults to  `" / "`):\n            Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].\n        doc_sep (`str`, *optional*, defaults to  `" // "`):\n            Separator inserted between the text of the retrieved document and the original input when calling\n            [`RagRetriever`].\n        n_docs (`int`, *optional*, defaults to 5):\n            Number of documents to retrieve.\n        max_combined_length (`int`, *optional*, defaults to 300):\n            Max length of contextualized input returned by [`~RagRetriever.__call__`].\n        retrieval_vector_size (`int`, *optional*, defaults to 768):\n            Dimensionality of the document embeddings indexed by [`RagRetriever`].\n        retrieval_batch_size (`int`, *optional*, defaults to 8):\n            Retrieval batch size, defined as the number of queries issues concurrently to the faiss index encapsulated\n            [`RagRetriever`].\n        dataset (`str`, *optional*, defaults to `"wiki_dpr"`):\n            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and ids\n            using `datasets.list_datasets()`).\n        dataset_split (`str`, *optional*, defaults to `"train"`)\n            Which split of the `dataset` to load.\n        index_name (`str`, *optional*, defaults to `"compressed"`)\n            The index name of the index associated with the `dataset`. One can choose between `"legacy"`, `"exact"` and\n            `"compressed"`.\n        index_path (`str`, *optional*)\n            The path to the serialized faiss index on disk.\n        passages_path (`str`, *optional*):\n            A path to text passages compatible with the faiss index. Required if using\n            [`~models.rag.retrieval_rag.LegacyIndex`]\n        use_dummy_dataset (`bool`, *optional*, defaults to `False`)\n            Whether to load a "dummy" variant of the dataset specified by `dataset`.\n        label_smoothing (`float`, *optional*, defaults to 0.0):\n            Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing\n            in the loss calculation. If set to 0, no label smoothing is performed.\n        do_marginalize (`bool`, *optional*, defaults to `False`):\n            If `True`, the logits are marginalized over all documents by making use of\n            `torch.nn.functional.log_softmax`.\n        reduce_loss (`bool`, *optional*, defaults to `False`):\n            Whether or not to reduce the NLL loss using the `torch.Tensor.sum` operation.\n        do_deduplication (`bool`, *optional*, defaults to `True`):\n            Whether or not to deduplicate the generations from different context documents for a given input. Has to be\n            set to `False` if used while training with distributed backend.\n        exclude_bos_score (`bool`, *optional*, defaults to `False`):\n            Whether or not to disregard the BOS token when computing the loss.\n        output_retrieved(`bool`, *optional*, defaults to `False`):\n            If set to `True`, `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and\n            `context_attention_mask` are returned. See returned tensors for more detail.\n        use_cache (`bool`, *optional*, defaults to `True`):\n            Whether or not the model should return the last key/values attentions (not used by all models).\n        forced_eos_token_id (`int`, *optional*):\n            The id of the token to force as the last generated token when `max_length` is reached. Usually set to\n            `eos_token_id`.\n'

@add_start_docstrings(A__					)
class 		_snake_case  (  A__					):
			_lowercase		:    Optional[int]			     =							'''rag'''
			_lowercase		:    Optional[int]			     =							True






			def __init__(       self		,					a=None		,					a=True		,					a=None		,					a=None		,					a=None		,					a=None		,					a=None		,					a=" / "		,					a=" // "		,					a=5		,					a=300		,					a=768		,					a=8		,					a="wiki_dpr"		,					a="train"		,					a="compressed"		,					a=None		,					a=None		,					a=False		,					a=False		,					a=0.0		,					a=True		,					a=False		,					a=False		,					a=False		,					a=True		,					a=None		,					**a		,					)      ->       Optional[Any]:
						super().__init__(
						    bos_token_id=a		,					pad_token_id=a		,					eos_token_id=a		,					decoder_start_token_id=a		,					forced_eos_token_id=a		,					is_encoder_decoder=a		,					prefix=a		,					vocab_size=a		,					**a		,					)
						assert (
						    "question_encoder" in kwargs and "generator" in kwargs
						), "Config has to be initialized with question_encoder and generator config"
						SCREAMING_SNAKE_CASE						= kwargs.pop('question_encoder')
						SCREAMING_SNAKE_CASE						= question_encoder_config.pop('model_type')
						SCREAMING_SNAKE_CASE						= kwargs.pop('generator')
						SCREAMING_SNAKE_CASE						= decoder_config.pop('model_type')

						from ..auto.configuration_auto import AutoConfig

						SCREAMING_SNAKE_CASE						= AutoConfig.for_model(a		,					**a)
						SCREAMING_SNAKE_CASE						= AutoConfig.for_model(a		,					**a)

						SCREAMING_SNAKE_CASE						= reduce_loss
						SCREAMING_SNAKE_CASE						= label_smoothing
						SCREAMING_SNAKE_CASE						= exclude_bos_score
						SCREAMING_SNAKE_CASE						= do_marginalize

						SCREAMING_SNAKE_CASE						= title_sep
						SCREAMING_SNAKE_CASE						= doc_sep
						SCREAMING_SNAKE_CASE						= n_docs
						SCREAMING_SNAKE_CASE						= max_combined_length

						SCREAMING_SNAKE_CASE						= dataset
						SCREAMING_SNAKE_CASE						= dataset_split
						SCREAMING_SNAKE_CASE						= index_name

						SCREAMING_SNAKE_CASE						= retrieval_vector_size
						SCREAMING_SNAKE_CASE						= retrieval_batch_size
						SCREAMING_SNAKE_CASE						= passages_path
						SCREAMING_SNAKE_CASE						= index_path
						SCREAMING_SNAKE_CASE						= use_dummy_dataset

						SCREAMING_SNAKE_CASE						= output_retrieved

						SCREAMING_SNAKE_CASE						= do_deduplication

						SCREAMING_SNAKE_CASE						= use_cache

						if self.forced_eos_token_id is None:
									SCREAMING_SNAKE_CASE						= getattr(self.generator		,					'forced_eos_token_id'		,					a)






			@classmethod
			def    SCREAMING_SNAKE_CASE__    (       cls		,					a		,					a		,					**a)      ->       PretrainedConfig:
						return cls(question_encoder=question_encoder_config.to_dict()		,					generator=generator_config.to_dict()		,					**a)







			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Union[str, Any]:
						SCREAMING_SNAKE_CASE						= copy.deepcopy(self.__dict__)
						SCREAMING_SNAKE_CASE						= self.question_encoder.to_dict()
						SCREAMING_SNAKE_CASE						= self.generator.to_dict()
						SCREAMING_SNAKE_CASE						= self.__class__.model_type
						return output




