


'''simple docstring'''



import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


_lowercase    :    Dict			     =  """<<<<<<< This should probably be modified because it mentions: """

_lowercase    :    Tuple			     =  """=======
>>>>>>>
"""

_lowercase    :    List[str]			     =  [
    """TextEncoderConfig""",
    """ByteTextEncoder""",
    """SubwordTextEncoder""",
    """encoder_config""",
    """maybe_build_from_corpus""",
    """manual_dir""",
]

_lowercase    :    List[str]			     =  [
    # (pattern, replacement)
    # Order is important here for some replacements
    (r"""tfds\.core""", r"""datasets"""),
    (r"""tf\.io\.gfile\.GFile""", r"""open"""),
    (r"""tf\.([\w\d]+)""", r"""datasets.Value('\1')"""),
    (r"""tfds\.features\.Text\(\)""", r"""datasets.Value('string')"""),
    (r"""tfds\.features\.Text\(""", r"""datasets.Value('string'),"""),
    (r"""features\s*=\s*tfds.features.FeaturesDict\(""", r"""features=datasets.Features("""),
    (r"""tfds\.features\.FeaturesDict\(""", r"""dict("""),
    (r"""The TensorFlow Datasets Authors""", r"""The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"""),
    (r"""tfds\.""", r"""datasets."""),
    (r"""dl_manager\.manual_dir""", r"""self.config.data_dir"""),
    (r"""self\.builder_config""", r"""self.config"""),
]


def    lowerCamelCase__   ( A     :    Namespace  ):




						'''simple docstring'''






						return ConvertCommand(args.tfds_path   ,     args.datasets_directory  )






class  UpperCamelCase__(     lowerCAmelCase ):




				@staticmethod
				def   a__(	lowerCAmelCase    :				ArgumentParser   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			parser.add_parser(
										    '''convert'''   ,					help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''   ,					)
										train_parser.add_argument(
										    '''--tfds_path'''   ,					type=lowerCAmelCase   ,					required=lowerCAmelCase   ,					help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''   ,					)
										train_parser.add_argument(
										    '''--datasets_directory'''   ,					type=lowerCAmelCase   ,					required=lowerCAmelCase   ,					help='''Path to the HuggingFace Datasets folder.'''   )
										train_parser.set_defaults(func=lowerCAmelCase   )




				def __init__(	self    :				Optional[Any]   ,					lowerCAmelCase    :				str   ,					lowerCAmelCase    :				str   ,					*lowerCAmelCase    :				str   )->  Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			get_logger('''datasets-cli/converting'''   )

										UpperCAmelCase         =			tfds_path
										UpperCAmelCase         =			datasets_directory





				def   a__(	self    :				List[Any]   )->  List[str]:

										"""simple docstring"""




										if os.path.isdir(self._tfds_path   ):
																UpperCAmelCase         =			os.path.abspath(self._tfds_path   )
										elif os.path.isfile(self._tfds_path   ):
																UpperCAmelCase         =			os.path.dirname(self._tfds_path   )
										else:
																raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.'''   )

										UpperCAmelCase         =			os.path.abspath(self._datasets_directory   )

										self._logger.info(F"""Converting datasets from {abs_tfds_path} to {abs_datasets_path}"""   )

										UpperCAmelCase         =			[]
										UpperCAmelCase         =			[]
										UpperCAmelCase         =			{}

										if os.path.isdir(self._tfds_path   ):
																UpperCAmelCase         =			os.listdir(lowerCAmelCase   )
										else:
																UpperCAmelCase         =			[os.path.basename(self._tfds_path   )]

										for f_name in file_names:
																self._logger.info(F"""Looking at file {f_name}"""   )
																UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					lowerCAmelCase   )
																UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					lowerCAmelCase   )

																if not os.path.isfile(lowerCAmelCase   ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
																						self._logger.info('''Skipping file'''   )
																						continue

																with open(lowerCAmelCase   ,					encoding='''utf-8'''   ) as f:
																						UpperCAmelCase         =			f.readlines()

																UpperCAmelCase         =			[]
																UpperCAmelCase         =			False
																UpperCAmelCase         =			False
																UpperCAmelCase         =			[]
																for line in lines:
																						UpperCAmelCase         =			line

																						# Convert imports
																						if "import tensorflow.compat.v2 as tf" in out_line:
																												continue
																						elif "@tfds.core" in out_line:
																												continue
																						elif "builder=self" in out_line:
																												continue
																						elif "import tensorflow_datasets.public_api as tfds" in out_line:
																												UpperCAmelCase         =			'''import datasets\n'''
																						elif "import tensorflow" in out_line:
																												# order is important here
																												UpperCAmelCase         =			''''''
																												continue
																						elif "from absl import logging" in out_line:
																												UpperCAmelCase         =			'''from datasets import logging\n'''
																						elif "getLogger" in out_line:
																												UpperCAmelCase         =			out_line.replace('''getLogger'''   ,					'''get_logger'''   )
																						elif any(expression in out_line for expression in TO_HIGHLIGHT   ):
																												UpperCAmelCase         =			True
																												UpperCAmelCase         =			list(filter(lambda lowerCAmelCase   : e in out_line   ,					lowerCAmelCase   )   )
																												out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(lowerCAmelCase   ) + '''\n'''   )
																												out_lines.append(lowerCAmelCase   )
																												out_lines.append(lowerCAmelCase   )
																												continue
																						else:
																												for pattern, replacement in TO_CONVERT:
																																		UpperCAmelCase         =			re.sub(lowerCAmelCase   ,					lowerCAmelCase   ,					lowerCAmelCase   )

                # Take care of saving utilities (to later move them together with main script)
																						if "tensorflow_datasets" in out_line:
																												UpperCAmelCase         =			re.match(R'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''   ,					lowerCAmelCase   )
																												tfds_imports.extend(imp.strip() for imp in match.group(1   ).split(''','''   )   )
																												UpperCAmelCase         =			'''from . import ''' + match.group(1   )

																						# Check we have not forget anything
																						if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
																												raise ValueError(F"""Error converting {out_line.strip()}"""   )

																						if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
																												UpperCAmelCase         =			True
																						out_lines.append(lowerCAmelCase   )

																if is_builder or "wmt" in f_name:
																						# We create a new directory for each dataset
																						UpperCAmelCase         =			f_name.replace('''.py'''   ,					''''''   )
																						UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					lowerCAmelCase   )
																						UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					lowerCAmelCase   )
																						os.makedirs(lowerCAmelCase   ,					exist_ok=lowerCAmelCase   )
																						self._logger.info(F"""Adding directory {output_dir}"""   )
																						imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}   )
																else:
																						# Utilities will be moved at the end
																						utils_files.append(lowerCAmelCase   )

																if needs_manual_update:
																						with_manual_update.append(lowerCAmelCase   )

																with open(lowerCAmelCase   ,					'''w'''   ,					encoding='''utf-8'''   ) as f:
																						f.writelines(lowerCAmelCase   )
																self._logger.info(F"""Converted in {output_file}"""   )

										for utils_file in utils_files:
																try:
																						UpperCAmelCase         =			os.path.basename(lowerCAmelCase   )
																						UpperCAmelCase         =			imports_to_builder_map[f_name.replace('''.py'''   ,					''''''   )]
																						self._logger.info(F"""Moving {dest_folder} to {utils_file}"""   )
																						shutil.copy(lowerCAmelCase   ,					lowerCAmelCase   )
																except KeyError:
																						self._logger.error(F"""Cannot find destination folder for {utils_file}. Please copy manually."""   )

										if with_manual_update:
																for file_path in with_manual_update:
																						self._logger.warning(
																						    F"""You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'."""   )









'''simple docstring'''



# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase    :    int			     =  {
    """configuration_xmod""": [
        """XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XmodConfig""",
        """XmodOnnxConfig""",
    ],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowercase    :    str			     =  [
	    """XMOD_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """XmodForCausalLM""",
	    """XmodForMaskedLM""",
	    """XmodForMultipleChoice""",
	    """XmodForQuestionAnswering""",
	    """XmodForSequenceClassification""",
	    """XmodForTokenClassification""",
	    """XmodModel""",
	    """XmodPreTrainedModel""",
	]

if TYPE_CHECKING:
	from .configuration_xmod import XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP, XmodConfig, XmodOnnxConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_xmod import (
		    XMOD_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XmodForCausalLM,
		    XmodForMaskedLM,
		    XmodForMultipleChoice,
		    XmodForQuestionAnswering,
		    XmodForSequenceClassification,
		    XmodForTokenClassification,
		    XmodModel,
		    XmodPreTrainedModel,
		)

else:
	import sys

	_lowercase    :    List[Any]			     =  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





