



import os
import sys

UpperCAmelCase_						=os.path.join(os.path.dirname(__file__), """src""")
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


UpperCAmelCase_						=[
    """torch""",
    """numpy""",
    """tokenizers""",
    """filelock""",
    """requests""",
    """tqdm""",
    """regex""",
    """sentencepiece""",
    """sacremoses""",
    """importlib_metadata""",
    """huggingface_hub""",
]



@add_start_docstrings(AutoConfig.__doc__							)
def       UpperCAmelCase       (     *_snake_case						, **_snake_case							):
							return AutoConfig.from_pretrained(*_snake_case						, **_snake_case							)



@add_start_docstrings(AutoTokenizer.__doc__							)
def       UpperCAmelCase       (     *_snake_case						, **_snake_case							):
							return AutoTokenizer.from_pretrained(*_snake_case						, **_snake_case							)



@add_start_docstrings(AutoModel.__doc__							)
def       UpperCAmelCase       (     *_snake_case						, **_snake_case							):
							return AutoModel.from_pretrained(*_snake_case						, **_snake_case							)



@add_start_docstrings(AutoModelForCausalLM.__doc__							)
def       UpperCAmelCase       (     *_snake_case						, **_snake_case							):
							return AutoModelForCausalLM.from_pretrained(*_snake_case						, **_snake_case							)



@add_start_docstrings(AutoModelForMaskedLM.__doc__							)
def       UpperCAmelCase       (     *_snake_case						, **_snake_case							):
							return AutoModelForMaskedLM.from_pretrained(*_snake_case						, **_snake_case							)



@add_start_docstrings(AutoModelForSequenceClassification.__doc__							)
def       UpperCAmelCase       (     *_snake_case						, **_snake_case							):
							return AutoModelForSequenceClassification.from_pretrained(*_snake_case						, **_snake_case							)



@add_start_docstrings(AutoModelForQuestionAnswering.__doc__							)
def       UpperCAmelCase       (     *_snake_case						, **_snake_case							):
							return AutoModelForQuestionAnswering.from_pretrained(*_snake_case						, **_snake_case							)









from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_ = None       , UpperCAmelCase_ = None       , UpperCAmelCase_ = True       , UpperCAmelCase_ = None       , UpperCAmelCase_ = False       , UpperCAmelCase_ = None       , UpperCAmelCase_ = True       , UpperCAmelCase_ = "arrow"       , **UpperCAmelCase_       , ):
											super().__init__(
											    split=UpperCAmelCase_       , features=UpperCAmelCase_       , cache_dir=UpperCAmelCase_       , keep_in_memory=UpperCAmelCase_       , streaming=UpperCAmelCase_       , **UpperCAmelCase_       , )
											lowerCAmelCase   =		load_from_cache_file
											lowerCAmelCase   =		file_format
											lowerCAmelCase   =		Spark(
											    df=UpperCAmelCase_       , features=UpperCAmelCase_       , cache_dir=UpperCAmelCase_       , working_dir=UpperCAmelCase_       , **UpperCAmelCase_       , )






				def       __snake_case		( self						):
											if self.streaming:
																		return self.builder.as_streaming_dataset(split=self.split						)
											lowerCAmelCase   =		None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
											self.builder.download_and_prepare(
											    download_mode=UpperCAmelCase_       , file_format=self._file_format       , )
											return self.builder.as_dataset(split=self.split						)



