




import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast




@dataclass
class 			UpperCAmelCase_      (   datasets.BuilderConfig       ):




							'''simple docstring'''



							a__						  =     None





class 			UpperCAmelCase_      (   datasets.ArrowBasedBuilder       ):




							'''simple docstring'''



							a__						  =     PandasConfig





							def    _UpperCAmelCase     (					self       :	List[Any]			)			->					Any:
										return datasets.DatasetInfo(features=self.config.features			)





							def    _UpperCAmelCase     (					self       :	Tuple       ,     a       :	Union[str, Any]			)			->					Union[str, Any]:
										if not self.config.data_files:
													raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""			)
										SCREAMING_SNAKE_CASE           =					dl_manager.download_and_extract(self.config.data_files			)
										if isinstance(a       ,     (str, list, tuple)			):
													SCREAMING_SNAKE_CASE           =					data_files
													if isinstance(a       ,     a			):
																SCREAMING_SNAKE_CASE           =					[files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													SCREAMING_SNAKE_CASE           =					[dl_manager.iter_files(a			) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,     gen_kwargs={"""files""": files}			)]
										SCREAMING_SNAKE_CASE           =					[]
										for split_name, files in data_files.items():
													if isinstance(a       ,     a			):
																SCREAMING_SNAKE_CASE           =					[files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													SCREAMING_SNAKE_CASE           =					[dl_manager.iter_files(a			) for file in files]
													splits.append(datasets.SplitGenerator(name=a       ,     gen_kwargs={"""files""": files}			)			)
										return splits





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	pa.Table			)			->					pa.Table:
										if self.config.features is not None:
													# more expensive cast to support nested features with keys in a different order
													# allows str <-> int/float or str to Audio for example
													SCREAMING_SNAKE_CASE           =					table_cast(a       ,     self.config.features.arrow_schema			)
										return pa_table







							def    _UpperCAmelCase     (					self       :	List[Any]       ,     a       :	Optional[Any]			)			->					int:
										for i, file in enumerate(itertools.chain.from_iterable(a			)			):
													with open(a       ,     """rb"""			) as f:
																SCREAMING_SNAKE_CASE           =					pa.Table.from_pandas(pd.read_pickle(a			)			)
																yield i, self._cast_table(a			)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__A						:							Dict							=  {
    """configuration_nllb_moe""": [
        """NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """NllbMoeConfig""",
    ]
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A						:							List[Any]							=  [
		    """NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """NllbMoeForConditionalGeneration""",
		    """NllbMoeModel""",
		    """NllbMoePreTrainedModel""",
		    """NllbMoeTop2Router""",
		    """NllbMoeSparseMLP""",
		]


if TYPE_CHECKING:
		from .configuration_nllb_moe import (
		    NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    NllbMoeConfig,
		)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_nllb_moe import (
				    NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
				    NllbMoeForConditionalGeneration,
				    NllbMoeModel,
				    NllbMoePreTrainedModel,
				    NllbMoeSparseMLP,
				    NllbMoeTopaRouter,
				)


else:
		import sys

		__A						:							Optional[Any]							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






