


'''simple docstring'''



import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


_lowercase    :    List[Any]			     =  datasets.utils.logging.get_logger(__name__)

_lowercase    :    Dict			     =  ['''names''', '''prefix''']
_lowercase    :    Tuple			     =  ['''warn_bad_lines''', '''error_bad_lines''', '''mangle_dupe_cols''']
_lowercase    :    Tuple			     =  ['''encoding_errors''', '''on_bad_lines''']
_lowercase    :    int			     =  ['''date_format''']






@dataclass
class  UpperCamelCase__(     datasets.BuilderConfig ):
				__magic_name__       :   str							= ","
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   Optional[Union[int, List[int], str]]							= "infer"
				__magic_name__       :   Optional[List[str]]							= None
				__magic_name__       :   Optional[List[str]]							= None
				__magic_name__       :   Optional[Union[int, str, List[int], List[str]]]							= None
				__magic_name__       :   Optional[Union[List[int], List[str]]]							= None
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   bool							= True
				__magic_name__       :   Optional[Literal["c", "python", "pyarrow"]]							= None
				__magic_name__       :   Dict[Union[int, str], Callable[[Any], Any]]							= None
				__magic_name__       :   Optional[list]							= None
				__magic_name__       :   Optional[list]							= None
				__magic_name__       :   bool							= False
				__magic_name__       :   Optional[Union[int, List[int]]]							= None
				__magic_name__       :   Optional[int]							= None
				__magic_name__       :   Optional[Union[str, List[str]]]							= None
				__magic_name__       :   bool							= True
				__magic_name__       :   bool							= True
				__magic_name__       :   bool							= False
				__magic_name__       :   bool							= True
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   str							= "."
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   str							= '"'
				__magic_name__       :   int							= 0
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   bool							= True
				__magic_name__       :   bool							= True
				__magic_name__       :   int							= 0
				__magic_name__       :   bool							= True
				__magic_name__       :   bool							= False
				__magic_name__       :   Optional[str]							= None
				__magic_name__       :   int							= 1_0000
				__magic_name__       :   Optional[datasets.Features]							= None
				__magic_name__       :   Optional[str]							= "strict"
				__magic_name__       :   Literal["error", "warn", "skip"]							= "error"
				__magic_name__       :   Optional[str]							= None




				def   a__(	self    :				List[Any]   )->  Tuple:

										"""simple docstring"""




										if self.delimiter is not None:
																UpperCAmelCase         =			self.delimiter
										if self.column_names is not None:
																UpperCAmelCase         =			self.column_names





				@property
				def   a__(	self    :				int   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			{
										    '''sep''': self.sep,
										    '''header''': self.header,
										    '''names''': self.names,
										    '''index_col''': self.index_col,
										    '''usecols''': self.usecols,
										    '''prefix''': self.prefix,
										    '''mangle_dupe_cols''': self.mangle_dupe_cols,
										    '''engine''': self.engine,
										    '''converters''': self.converters,
										    '''true_values''': self.true_values,
										    '''false_values''': self.false_values,
										    '''skipinitialspace''': self.skipinitialspace,
										    '''skiprows''': self.skiprows,
										    '''nrows''': self.nrows,
										    '''na_values''': self.na_values,
										    '''keep_default_na''': self.keep_default_na,
										    '''na_filter''': self.na_filter,
										    '''verbose''': self.verbose,
										    '''skip_blank_lines''': self.skip_blank_lines,
										    '''thousands''': self.thousands,
										    '''decimal''': self.decimal,
										    '''lineterminator''': self.lineterminator,
										    '''quotechar''': self.quotechar,
										    '''quoting''': self.quoting,
										    '''escapechar''': self.escapechar,
										    '''comment''': self.comment,
										    '''encoding''': self.encoding,
										    '''dialect''': self.dialect,
										    '''error_bad_lines''': self.error_bad_lines,
										    '''warn_bad_lines''': self.warn_bad_lines,
										    '''skipfooter''': self.skipfooter,
										    '''doublequote''': self.doublequote,
										    '''memory_map''': self.memory_map,
										    '''float_precision''': self.float_precision,
										    '''chunksize''': self.chunksize,
										    '''encoding_errors''': self.encoding_errors,
										    '''on_bad_lines''': self.on_bad_lines,
										    '''date_format''': self.date_format,
										}

										# some kwargs must not be passed if they don't have a default value
										# some others are deprecated and we can also not pass them if they are the default value
										for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
																if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()   ,					__SCREAMING_SNAKE_CASE   ):
																						del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
										if not (datasets.config.PANDAS_VERSION.major >= 2):
																for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
																						del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
										if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
																for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
																						del pd_read_csv_kwargs[pd_read_csv_parameter]

										return pd_read_csv_kwargs







class  UpperCamelCase__(     datasets.ArrowBasedBuilder ):
				__magic_name__       :   List[Any]							= CsvConfig




				def   a__(	self    :				List[Any]   )->  Dict:

										"""simple docstring"""




										return datasets.DatasetInfo(features=self.config.features   )




				def   a__(	self    :				List[str]   ,					lowerCAmelCase    :				str   )->  List[str]:

										"""simple docstring"""




										if not self.config.data_files:
																raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
										UpperCAmelCase         =			dl_manager.download_and_extract(self.config.data_files   )
										if isinstance(__SCREAMING_SNAKE_CASE   ,					(str, list, tuple)   ):
																UpperCAmelCase         =			data_files
																if isinstance(__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE   ):
																						UpperCAmelCase         =			[files]
																UpperCAmelCase         =			[dl_manager.iter_files(__SCREAMING_SNAKE_CASE   ) for file in files]
																return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,					gen_kwargs={'''files''': files}   )]
										UpperCAmelCase         =			[]
										for split_name, files in data_files.items():
																if isinstance(__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE   ):
																						UpperCAmelCase         =			[files]
																UpperCAmelCase         =			[dl_manager.iter_files(__SCREAMING_SNAKE_CASE   ) for file in files]
																splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE   ,					gen_kwargs={'''files''': files}   )   )
										return splits




				def   a__(	self    :				Union[str, Any]   ,					lowerCAmelCase    :				pa.Table   )->  Any:

										"""simple docstring"""




										if self.config.features is not None:
																UpperCAmelCase         =			self.config.features.arrow_schema
																if all(not require_storage_cast(__SCREAMING_SNAKE_CASE   ) for feature in self.config.features.values()   ):
																						# cheaper cast
																						UpperCAmelCase         =			pa.Table.from_arrays([pa_table[field.name] for field in schema]   ,					schema=__SCREAMING_SNAKE_CASE   )
																else:
																						# more expensive cast; allows str <-> int/float or str to Audio for example
																						UpperCAmelCase         =			table_cast(__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE   )
										return pa_table





				def   a__(	self    :				int   ,					lowerCAmelCase    :				Optional[Any]   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			self.config.features.arrow_schema if self.config.features else None
										# dtype allows reading an int column as str
										UpperCAmelCase         =			(
										    {
										        name: dtype.to_pandas_dtype() if not require_storage_cast(__SCREAMING_SNAKE_CASE   ) else object
										        for name, dtype, feature in zip(schema.names   ,					schema.types   ,					self.config.features.values()   )
										    }
										    if schema is not None
										    else None
										)
										for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE   )   ):
																UpperCAmelCase         =			pd.read_csv(__SCREAMING_SNAKE_CASE   ,					iterator=__SCREAMING_SNAKE_CASE   ,					dtype=__SCREAMING_SNAKE_CASE   ,					**self.config.pd_read_csv_kwargs   )
																try:
																						for batch_idx, df in enumerate(__SCREAMING_SNAKE_CASE   ):
																												UpperCAmelCase         =			pa.Table.from_pandas(__SCREAMING_SNAKE_CASE   )
																												# Uncomment for debugging (will print the Arrow table size and elements)
																												# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																												# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																												yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE   )
																except ValueError as e:
																						logger.error(F"""Failed to read file \'{file}\' with error {type(__SCREAMING_SNAKE_CASE   )}: {e}"""   )
																						raise









'''simple docstring'''



import argparse
import os
import re

import packaging.version


_lowercase    :    Optional[int]			     =  """examples/"""
_lowercase    :    str			     =  {
    """examples""": (re.compile(r"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(r"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(r"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), r"""\1version=\"VERSION\","""),
    """doc""": (re.compile(r"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
_lowercase    :    Dict			     =  {
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
_lowercase    :    List[Any]			     =  """README.md"""


def    lowerCamelCase__   ( A     :    int   ,     A     :    str   ,     A     :    Optional[Any]  ):




						'''simple docstring'''






						with open(A   ,     '''r'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												UpperCAmelCase         =			f.read()
						UpperCAmelCase       ,       UpperCAmelCase         =			REPLACE_PATTERNS[pattern]
						UpperCAmelCase         =			replace.replace('''VERSION'''   ,     A  )
						UpperCAmelCase         =			re_pattern.sub(A   ,     A  )
						with open(A   ,     '''w'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												f.write(A  )


def    lowerCamelCase__   ( A     :    Optional[int]  ):




						'''simple docstring'''






						for folder, directories, fnames in os.walk(A  ):
												# Removing some of the folders with non-actively maintained examples from the walk
												if "research_projects" in directories:
																		directories.remove('''research_projects'''  )
												if "legacy" in directories:
																		directories.remove('''legacy'''  )
												for fname in fnames:
																		if fname.endswith('''.py'''  ):
																								update_version_in_file(os.path.join(A   ,     A  )   ,     A   ,     pattern='''examples'''  )


def    lowerCamelCase__   ( A     :    str   ,     A     :    Dict=False  ):




						'''simple docstring'''






						for pattern, fname in REPLACE_FILES.items():
												update_version_in_file(A   ,     A   ,     A  )
						if not patch:
												update_version_in_examples(A  )


def    lowerCamelCase__   ( ):




						'''simple docstring'''






						UpperCAmelCase         =			'''🤗 Transformers currently provides the following architectures'''
						UpperCAmelCase         =			'''1. Want to contribute a new model?'''
						with open(A   ,     '''r'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												UpperCAmelCase         =			f.readlines()

						# Find the start of the list.
						UpperCAmelCase         =			0
						while not lines[start_index].startswith(_start_prompt  ):
												start_index += 1
						start_index += 1

						UpperCAmelCase         =			start_index
						# Update the lines in the model list.
						while not lines[index].startswith(_end_prompt  ):
												if lines[index].startswith('''1.'''  ):
																		UpperCAmelCase         =			lines[index].replace(
																		    '''https://huggingface.co/docs/transformers/main/model_doc'''   ,     '''https://huggingface.co/docs/transformers/model_doc'''   ,     )
												index += 1

						with open(A   ,     '''w'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												f.writelines(A  )


def    lowerCamelCase__   ( ):




						'''simple docstring'''






						with open(REPLACE_FILES['''init''']   ,     '''r'''  ) as f:
												UpperCAmelCase         =			f.read()
						UpperCAmelCase         =			REPLACE_PATTERNS['''init'''][0].search(A  ).groups()[0]
						return packaging.version.parse(A  )


def    lowerCamelCase__   ( A     :    Tuple=False  ):




						'''simple docstring'''






						UpperCAmelCase         =			get_version()
						if patch and default_version.is_devrelease:
												raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''  )
						if default_version.is_devrelease:
												UpperCAmelCase         =			default_version.base_version
						elif patch:
												UpperCAmelCase         =			f"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
						else:
												UpperCAmelCase         =			f"""{default_version.major}.{default_version.minor + 1}.0"""

						# Now let's ask nicely if that's the right one.
						UpperCAmelCase         =			input(f"""Which version are you releasing? [{default_version}]"""  )
						if len(A  ) == 0:
												UpperCAmelCase         =			default_version

						print(f"""Updating version to {version}."""  )
						global_version_update(A   ,     patch=A  )
						if not patch:
												print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''  )
												clean_main_ref_in_model_list()


def    lowerCamelCase__   ( ):




						'''simple docstring'''






						UpperCAmelCase         =			get_version()
						UpperCAmelCase         =			f"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
						UpperCAmelCase         =			current_version.base_version

						# Check with the user we got that right.
						UpperCAmelCase         =			input(f"""Which version are we developing now? [{dev_version}]"""  )
						if len(A  ) == 0:
												UpperCAmelCase         =			dev_version

						print(f"""Updating version to {version}."""  )
						global_version_update(A  )
						print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''  )
						clean_main_ref_in_model_list()


if __name__ == "__main__":
	_lowercase    :    Optional[Any]			     =  argparse.ArgumentParser()
	parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
	parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
	_lowercase    :    Union[str, Any]			     =  parser.parse_args()
	if not args.post_release:
		pre_release_work(patch=args.patch)
	elif args.patch:
		print("""Nothing to do after a patch :-)""")
	else:
		post_release_work()





