


'''simple docstring'''



import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


_lowercase    :    str			     =  [
    {"dataset": "wikipedia", "config_name": "20220301.de"},
    {"dataset": "wikipedia", "config_name": "20220301.en"},
    {"dataset": "wikipedia", "config_name": "20220301.fr"},
    {"dataset": "wikipedia", "config_name": "20220301.frr"},
    {"dataset": "wikipedia", "config_name": "20220301.it"},
    {"dataset": "wikipedia", "config_name": "20220301.simple"},
    {"dataset": "snli", "config_name": "plain_text"},
    {"dataset": "eli5", "config_name": "LFQA_reddit"},
    {"dataset": "wiki40b", "config_name": "en"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.compressed"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.no_index"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.multiset.no_index"},
    {"dataset": "natural_questions", "config_name": "default"},
]


def    lowerCamelCase__   ( A     :    int=True  ):




						'''simple docstring'''






						if with_config:
												return [
												    {
												        "testcase_name": d["dataset"] + "/" + d["config_name"],
												        "dataset": d["dataset"],
												        "config_name": d["config_name"],
												    }
												    for d in DATASETS_ON_HF_GCP
												]
						else:
												return [
												    {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
												]






@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=__UpperCAmelCase ) )
class  UpperCamelCase__(     __UpperCAmelCase ):
				__magic_name__       :   str							= None
				__magic_name__       :   str							= None




				def   a__(	self    :				Any   ,					lowerCAmelCase    :				Tuple   ,					lowerCAmelCase    :				int   )->  Dict:

										"""simple docstring"""




										with TemporaryDirectory() as tmp_dir:
																UpperCAmelCase         =			dataset_module_factory(lowerCAmelCase_   ,					cache_dir=lowerCAmelCase_   )

																UpperCAmelCase         =			import_main_class(dataset_module.module_path   ,					dataset=lowerCAmelCase_   )

																UpperCAmelCase         =			builder_cls(
																    cache_dir=lowerCAmelCase_   ,					config_name=lowerCAmelCase_   ,					hash=dataset_module.hash   ,					)

																UpperCAmelCase         =			'''/'''.join(
																    [
																        HF_GCP_BASE_URL,
																        builder_instance._relative_data_dir(with_hash=lowerCAmelCase_   ).replace(os.sep   ,					'''/'''   ),
																        config.DATASET_INFO_FILENAME,
																    ]   )
																UpperCAmelCase         =			cached_path(lowerCAmelCase_   ,					cache_dir=lowerCAmelCase_   )
																self.assertTrue(os.path.exists(lowerCAmelCase_   )   )







@pytest.mark.integration
def    lowerCamelCase__   ( A     :    Union[str, Any]  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path_factory.mktemp('''test_hf_gcp'''  ) / '''test_wikipedia_simple'''
						UpperCAmelCase         =			dataset_module_factory('''wikipedia'''   ,     cache_dir=__lowerCAmelCase  )
						UpperCAmelCase         =			import_main_class(dataset_module.module_path  )
						UpperCAmelCase         =			builder_cls(
						    cache_dir=__lowerCAmelCase   ,     config_name='''20220301.frr'''   ,     hash=dataset_module.hash   ,     )
						# use the HF cloud storage, not the original download_and_prepare that uses apache-beam
						UpperCAmelCase         =			None
						builder_instance.download_and_prepare()
						UpperCAmelCase         =			builder_instance.as_dataset()
						assert ds




@pytest.mark.integration
def    lowerCamelCase__   ( A     :    Union[str, Any]  ):




						'''simple docstring'''






						UpperCAmelCase         =			dataset_module_factory('''wikipedia'''   ,     cache_dir=__lowerCAmelCase  )
						UpperCAmelCase         =			import_main_class(dataset_module.module_path   ,     dataset=__lowerCAmelCase  )
						UpperCAmelCase         =			builder_cls(
						    cache_dir=__lowerCAmelCase   ,     config_name='''20220301.frr'''   ,     hash=dataset_module.hash   ,     )
						UpperCAmelCase         =			builder_instance.as_streaming_dataset()
						assert ds
						assert isinstance(__lowerCAmelCase   ,     __lowerCAmelCase  )
						assert "train" in ds
						assert isinstance(ds['''train''']   ,     __lowerCAmelCase  )
						assert next(iter(ds['''train''']  )  )







'''simple docstring'''



# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase    :    int			     =  {
    """configuration_xmod""": [
        """XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XmodConfig""",
        """XmodOnnxConfig""",
    ],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowercase    :    str			     =  [
	    """XMOD_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """XmodForCausalLM""",
	    """XmodForMaskedLM""",
	    """XmodForMultipleChoice""",
	    """XmodForQuestionAnswering""",
	    """XmodForSequenceClassification""",
	    """XmodForTokenClassification""",
	    """XmodModel""",
	    """XmodPreTrainedModel""",
	]

if TYPE_CHECKING:
	from .configuration_xmod import XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP, XmodConfig, XmodOnnxConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_xmod import (
		    XMOD_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XmodForCausalLM,
		    XmodForMaskedLM,
		    XmodForMultipleChoice,
		    XmodForQuestionAnswering,
		    XmodForSequenceClassification,
		    XmodForTokenClassification,
		    XmodModel,
		    XmodPreTrainedModel,
		)

else:
	import sys

	_lowercase    :    List[Any]			     =  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





