






import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
						from .features import FeatureType


__UpperCamelCase						,					__UpperCamelCase						,					__UpperCamelCase      :      Dict     						=						False, False, False


@dataclass
class        __UpperCamelCase    :
	__snake_case	:Optional[int]   =       None
	__snake_case	:bool   =       True
	__snake_case	:bool   =       True
	__snake_case	:Optional[str]   =       None
	# Automatically constructed
	__snake_case	:ClassVar[str]   =       "dict"
	__snake_case	:ClassVar[Any]   =       pa.struct({'bytes': pa.binary(), 'path': pa.string()}   )
	__snake_case	:str   =       field(default='Audio'   ,			init=lowerCAmelCase__   ,			repr=lowerCAmelCase__   )







	def __call__(  self      :							Dict			)		-> Optional[int]:
		"""simple docstring"""







		return self.pa_type







	def  _a   (  self      :							Tuple       ,				_lowerCAmelCase      :							Optional[Any]			)		-> dict:
		"""simple docstring"""







		try:
			import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
		except ImportError as err:
			raise ImportError("""To support encoding audio data, please install \'soundfile\'."""			) from err
		if isinstance(_lowerCamelCase       ,				_lowerCamelCase			):
			return {"bytes": None, "path": value}
		elif isinstance(_lowerCamelCase       ,				_lowerCamelCase			):
			return {"bytes": value, "path": None}
		elif "array" in value:
			# convert the audio array to wav bytes
			__lowercase			       =  BytesIO()
			sf.write(_lowerCamelCase       ,				value["""array"""]       ,				value["""sampling_rate"""]       ,				format="""wav"""			)
			return {"bytes": buffer.getvalue(), "path": None}
		elif value.get("""path"""			) is not None and os.path.isfile(value["""path"""]			):
			# we set "bytes": None to not duplicate the data if they're already available locally
			if value["path"].endswith("""pcm"""			):
				# "PCM" only has raw audio bytes
				if value.get("""sampling_rate"""			) is None:
					# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
					raise KeyError("""To use PCM files, please specify a \'sampling_rate\' in Audio object"""			)
				if value.get("""bytes"""			):
					# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
					__lowercase			       =  np.frombuffer(value["""bytes"""]       ,				dtype=np.intaa			).astype(np.floataa			) / 3_2767
				else:
					__lowercase			       =  np.memmap(value["""path"""]       ,				dtype="""h"""       ,				mode="""r"""			).astype(np.floataa			) / 3_2767

				__lowercase			       =  BytesIO(bytes()			)
				sf.write(_lowerCamelCase       ,				_lowerCamelCase       ,				value["""sampling_rate"""]       ,				format="""wav"""			)
				return {"bytes": buffer.getvalue(), "path": None}
			else:
				return {"bytes": None, "path": value.get("""path"""			)}
		elif value.get("""bytes"""			) is not None or value.get("""path"""			) is not None:
			# store the audio bytes, and path is used to infer the audio format using the file extension
			return {"bytes": value.get("""bytes"""			), "path": value.get("""path"""			)}
		else:
			raise ValueError(
			    F'An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'			)







	def  _a   (  self      :							Tuple       ,				_lowerCAmelCase      :							str       ,				_lowerCAmelCase      :							Dict = None			)		-> dict:
		"""simple docstring"""







		if not self.decode:
			raise RuntimeError("""Decoding is disabled for this feature. Please use Audio(decode=True) instead."""			)

		__lowercase       ,						__lowercase			       =  (value["""path"""], BytesIO(value["""bytes"""]			)) if value["""bytes"""] is not None else (value["""path"""], None)
		if path is None and file is None:
			raise ValueError(F'An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.'			)

		try:
			import librosa
			import soundfile as sf
		except ImportError as err:
			raise ImportError("""To support decoding audio files, please install \'librosa\' and \'soundfile\'."""			) from err

		__lowercase			       =  xsplitext(_lowerCamelCase			)[1][1:].lower() if path is not None else None
		if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
			raise RuntimeError(
			    """Decoding \'opus\' files requires system library \'libsndfile\'>=1.0.31, """
			    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """			)
		elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
			raise RuntimeError(
			    """Decoding \'mp3\' files requires system library \'libsndfile\'>=1.1.0, """
			    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """			)

		if file is None:
			__lowercase			       =  token_per_repo_id or {}
			__lowercase			       =  path.split("""::"""			)[-1]
			try:
				__lowercase			       =  string_to_dict(_lowerCamelCase       ,				config.HUB_DATASETS_URL			)["""repo_id"""]
				__lowercase			       =  token_per_repo_id[repo_id]
			except (ValueError, KeyError):
				__lowercase			       =  None

			with xopen(_lowerCamelCase       ,				"""rb"""       ,				use_auth_token=_lowerCamelCase			) as f:
				__lowercase       ,						__lowercase			       =  sf.read(_lowerCamelCase			)

		else:
			__lowercase       ,						__lowercase			       =  sf.read(_lowerCamelCase			)

		__lowercase			       =  array.T
		if self.mono:
			__lowercase			       =  librosa.to_mono(_lowerCamelCase			)
		if self.sampling_rate and self.sampling_rate != sampling_rate:
			__lowercase			       =  librosa.resample(_lowerCamelCase       ,				orig_sr=_lowerCamelCase       ,				target_sr=self.sampling_rate			)
			__lowercase			       =  self.sampling_rate

		return {"path": path, "array": array, "sampling_rate": sampling_rate}







	def  _a   (  self      :							int			)		-> Union["FeatureType", Dict[str, "FeatureType"]]:
		"""simple docstring"""







		from .features import Value

		if self.decode:
			raise ValueError("""Cannot flatten a decoded Audio feature."""			)
		return {
		    "bytes": Value("""binary"""			),
		    "path": Value("""string"""			),
		}







	def  _a   (  self      :							Tuple       ,				_lowerCAmelCase      :							Any			)		-> pa.StructArray:
		"""simple docstring"""







		if pa.types.is_string(storage.type			):
			__lowercase			       =  pa.array([None] * len(_lowerCamelCase			)       ,				type=pa.binary()			)
			__lowercase			       =  pa.StructArray.from_arrays([bytes_array, storage]       ,				["""bytes""", """path"""]       ,				mask=storage.is_null()			)
		elif pa.types.is_binary(storage.type			):
			__lowercase			       =  pa.array([None] * len(_lowerCamelCase			)       ,				type=pa.string()			)
			__lowercase			       =  pa.StructArray.from_arrays([storage, path_array]       ,				["""bytes""", """path"""]       ,				mask=storage.is_null()			)
		elif pa.types.is_struct(storage.type			) and storage.type.get_all_field_indices("""array"""			):
			__lowercase			       =  pa.array([Audio().encode_example(_lowerCamelCase			) if x is not None else None for x in storage.to_pylist()]			)
		elif pa.types.is_struct(storage.type			):
			if storage.type.get_field_index("""bytes"""			) >= 0:
				__lowercase			       =  storage.field("""bytes"""			)
			else:
				__lowercase			       =  pa.array([None] * len(_lowerCamelCase			)       ,				type=pa.binary()			)
			if storage.type.get_field_index("""path"""			) >= 0:
				__lowercase			       =  storage.field("""path"""			)
			else:
				__lowercase			       =  pa.array([None] * len(_lowerCamelCase			)       ,				type=pa.string()			)
			__lowercase			       =  pa.StructArray.from_arrays([bytes_array, path_array]       ,				["""bytes""", """path"""]       ,				mask=storage.is_null()			)
		return array_cast(_lowerCamelCase       ,				self.pa_type			)







	def  _a   (  self      :							List[Any]       ,				_lowerCAmelCase      :							Union[str, Any]			)		-> pa.StructArray:
		"""simple docstring"""







		@no_op_if_value_is_null
		def path_to_bytes(_lowerCAmelCase      :							Optional[Any]			):
			with xopen(_lowerCamelCase       ,				"""rb"""			) as f:
				__lowercase			       =  f.read()
			return bytes_

		__lowercase			       =  pa.array(
		    [
		        (path_to_bytes(x["""path"""]			) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
		        for x in storage.to_pylist()
		    ]       ,				type=pa.binary()       ,				)
		__lowercase			       =  pa.array(
		    [os.path.basename(_lowerCamelCase			) if path is not None else None for path in storage.field("""path"""			).to_pylist()]       ,				type=pa.string()       ,				)
		__lowercase			       =  pa.StructArray.from_arrays([bytes_array, path_array]       ,				["""bytes""", """path"""]       ,				mask=bytes_array.is_null()			)
		return array_cast(_lowerCamelCase       ,				self.pa_type			)









import os
import time
import warnings
from dataclasses import dataclass, field
from enum import Enum
from typing import List, Optional, Union

import torch
from filelock import FileLock
from torch.utils.data import Dataset

from ...tokenization_utils_base import PreTrainedTokenizerBase
from ...utils import logging
from ..processors.glue import glue_convert_examples_to_features, glue_output_modes, glue_processors
from ..processors.utils import InputFeatures


__UpperCamelCase      :      Any     						=						logging.get_logger(__name__)


@dataclass
class        __UpperCamelCase    :
	__snake_case	:str   =       field(metadata={'help': 'The name of the task to train on: ' + ', '.join(glue_processors.keys()   )}   )
	__snake_case	:str   =       field(
	    metadata={'help': 'The input data dir. Should contain the .tsv files (or other data files) for the task.'}   )
	__snake_case	:int   =       field(
	    default=1_2_8   ,			metadata={
	        'help': (
	            'The maximum total input sequence length after tokenization. Sequences longer '
	            'than this will be truncated, sequences shorter will be padded.'
	        )
	    }   ,			)
	__snake_case	:bool   =       field(
	    default=_lowerCAmelCase   ,			metadata={'help': 'Overwrite the cached training and evaluation sets'}   )







	def  _a   (  self      :							Dict			)		-> List[Any]:
		"""simple docstring"""







		__lowercase             =  self.task_name.lower()


class        __UpperCamelCase    (     _lowerCAmelCase   ):
	__snake_case	:Optional[int]   =       'train'
	__snake_case	:int   =       'dev'
	__snake_case	:Any   =       'test'


class        __UpperCamelCase    (     _lowerCAmelCase   ):
	__snake_case	:GlueDataTrainingArguments
	__snake_case	:str
	__snake_case	:List[InputFeatures]







	def __init__(  self      :							Dict       ,				_lowerCAmelCase      :							GlueDataTrainingArguments       ,				_lowerCAmelCase      :							PreTrainedTokenizerBase       ,				_lowerCAmelCase      :							Optional[int] = None       ,				_lowerCAmelCase      :							Union[str, Split] = Split.train       ,				_lowerCAmelCase      :							Optional[str] = None       ,				)		-> List[Any]:
		"""simple docstring"""







		warnings.warn(
		    """This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets """
		    """library. You can have a look at this example script for pointers: """
		    """https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.py"""       ,				_lowerCAmelCase       ,				)
		__lowercase             =  args
		__lowercase             =  glue_processors[args.task_name]()
		__lowercase             =  glue_output_modes[args.task_name]
		if isinstance(_lowerCAmelCase       ,				_lowerCAmelCase			):
			try:
				__lowercase             =  Split[mode]
			except KeyError:
				raise KeyError("""mode is not a valid split name"""			)
        # Load data features from cache or dataset file
		__lowercase             =  os.path.join(
		    cache_dir if cache_dir is not None else args.data_dir       ,				F'cached_{mode.value}_{tokenizer.__class__.__name__}_{args.max_seq_length}_{args.task_name}'       ,				)
		__lowercase             =  self.processor.get_labels()
		if args.task_name in ["mnli", "mnli-mm"] and tokenizer.__class__.__name__ in (
		    "RobertaTokenizer",
		    "RobertaTokenizerFast",
		    "XLMRobertaTokenizer",
		    "BartTokenizer",
		    "BartTokenizerFast",
		):
			# HACK(label indices are swapped in RoBERTa pretrained model)
			__lowercase       ,						__lowercase             =  label_list[2], label_list[1]
		__lowercase             =  label_list

		# Make sure only the first process in distributed training processes the dataset,
		# and the others will use the cache.
		__lowercase             =  cached_features_file + """.lock"""
		with FileLock(_lowerCAmelCase			):
			if os.path.exists(_lowerCAmelCase			) and not args.overwrite_cache:
				__lowercase             =  time.time()
				__lowercase             =  torch.load(_lowerCAmelCase			)
				logger.info(
				    F'Loading features from cached file {cached_features_file} [took %.3f s]'       ,				time.time() - start			)
			else:
				logger.info(F'Creating features from dataset file at {args.data_dir}'			)

				if mode == Split.dev:
					__lowercase             =  self.processor.get_dev_examples(args.data_dir			)
				elif mode == Split.test:
					__lowercase             =  self.processor.get_test_examples(args.data_dir			)
				else:
					__lowercase             =  self.processor.get_train_examples(args.data_dir			)
				if limit_length is not None:
					__lowercase             =  examples[:limit_length]
				__lowercase             =  glue_convert_examples_to_features(
				    _lowerCAmelCase       ,				_lowerCAmelCase       ,				max_length=args.max_seq_length       ,				label_list=_lowerCAmelCase       ,				output_mode=self.output_mode       ,				)
				__lowercase             =  time.time()
				torch.save(self.features       ,				_lowerCAmelCase			)
				# ^ This seems to take a lot of time so I want to investigate why and how we can improve.
				logger.info(
				    F'Saving features into cached file {cached_features_file} [took {time.time() - start:.3f} s]'			)







	def __len__(  self      :							Dict			)		-> Optional[int]:
		"""simple docstring"""







		return len(self.features			)







	def __getitem__(  self      :							Tuple       ,				_lowerCAmelCase      :							Optional[int]			)		-> InputFeatures:
		"""simple docstring"""







		return self.features[i]







	def  _a   (  self      :							str			)		-> int:
		"""simple docstring"""







		return self.label_list

