



import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
    from .features import FeatureType


lowercase_       ,       lowercase_       ,       lowercase_	       =			False, False, False






@dataclass
class 						__a :
 SCREAMING_SNAKE_CASE       =       None
 SCREAMING_SNAKE_CASE       =       True
 SCREAMING_SNAKE_CASE       =       True
 SCREAMING_SNAKE_CASE       =       None
 # Automatically constructed
 SCREAMING_SNAKE_CASE       =       "dict"
 SCREAMING_SNAKE_CASE       =       pa.struct({"bytes": pa.binary(), "path": pa.string()}				)
 SCREAMING_SNAKE_CASE       =       field(default="Audio"	,    init=_snake_case	,    repr=_snake_case				)



 def __call__(						self							:    List[str])->							List[Any]:
      return self.pa_type



 def 		UpperCamelCase							(						self							:    str				,	snake_case_							:    Union[str, bytes, dict])->							dict:
      try:
           import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
      except ImportError as err:
           raise ImportError("""To support encoding audio data, please install 'soundfile'.""") from err
      if isinstance(lowerCAmelCase__				,	lowerCAmelCase__):
           return {"bytes": None, "path": value}
      elif isinstance(lowerCAmelCase__				,	lowerCAmelCase__):
           return {"bytes": value, "path": None}
      elif "array" in value:
           # convert the audio array to wav bytes
           __lowerCAmelCase	   =BytesIO()
           sf.write(lowerCAmelCase__				,	value["""array"""]				,	value["""sampling_rate"""]				,	format="""wav""")
           return {"bytes": buffer.getvalue(), "path": None}
      elif value.get("""path""") is not None and os.path.isfile(value["""path"""]):
           # we set "bytes": None to not duplicate the data if they're already available locally
           if value["path"].endswith("""pcm"""):
                # "PCM" only has raw audio bytes
                if value.get("""sampling_rate""") is None:
                     # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
                     raise KeyError("""To use PCM files, please specify a 'sampling_rate' in Audio object""")
                if value.get("""bytes"""):
                     # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
                     __lowerCAmelCase	   =np.frombuffer(value["""bytes"""]				,	dtype=np.intaa).astype(np.floataa) / 3_27_67
                else:
                     __lowerCAmelCase	   =np.memmap(value["""path"""]				,	dtype="""h"""				,	mode="""r""").astype(np.floataa) / 3_27_67

                __lowerCAmelCase	   =BytesIO(bytes())
                sf.write(lowerCAmelCase__				,	lowerCAmelCase__				,	value["""sampling_rate"""]				,	format="""wav""")
                return {"bytes": buffer.getvalue(), "path": None}
           else:
                return {"bytes": None, "path": value.get("""path""")}
      elif value.get("""bytes""") is not None or value.get("""path""") is not None:
           # store the audio bytes, and path is used to infer the audio format using the file extension
           return {"bytes": value.get("""bytes"""), "path": value.get("""path""")}
      else:
           raise ValueError(
               F"""An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}.""")



 def 		UpperCamelCase							(						self							:    Any				,	snake_case_							:    dict				,	snake_case_							:    Optional[Dict[str, Union[str, bool, None]]] = None)->							dict:
      if not self.decode:
           raise RuntimeError("""Decoding is disabled for this feature. Please use Audio(decode=True) instead.""")

      __lowerCAmelCase   ,	__lowerCAmelCase	   =(value["""path"""], BytesIO(value["""bytes"""])) if value["""bytes"""] is not None else (value["""path"""], None)
      if path is None and file is None:
           raise ValueError(F"""An audio sample should have one of 'path' or 'bytes' but both are None in {value}.""")

      try:
           import librosa
           import soundfile as sf
      except ImportError as err:
           raise ImportError("""To support decoding audio files, please install 'librosa' and 'soundfile'.""") from err

      __lowerCAmelCase	   =xsplitext(lowerCAmelCase__)[1][1:].lower() if path is not None else None
      if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
           raise RuntimeError(
               """Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, """
               """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """)
      elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
           raise RuntimeError(
               """Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, """
               """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """)

      if file is None:
           __lowerCAmelCase	   =token_per_repo_id or {}
           __lowerCAmelCase	   =path.split("""::""")[-1]
           try:
                __lowerCAmelCase	   =string_to_dict(lowerCAmelCase__				,	config.HUB_DATASETS_URL)["""repo_id"""]
                __lowerCAmelCase	   =token_per_repo_id[repo_id]
           except (ValueError, KeyError):
                __lowerCAmelCase	   =None

           with xopen(lowerCAmelCase__				,	"""rb"""				,	use_auth_token=lowerCAmelCase__) as f:
                __lowerCAmelCase   ,	__lowerCAmelCase	   =sf.read(lowerCAmelCase__)

      else:
           __lowerCAmelCase   ,	__lowerCAmelCase	   =sf.read(lowerCAmelCase__)

      __lowerCAmelCase	   =array.T
      if self.mono:
           __lowerCAmelCase	   =librosa.to_mono(lowerCAmelCase__)
      if self.sampling_rate and self.sampling_rate != sampling_rate:
           __lowerCAmelCase	   =librosa.resample(lowerCAmelCase__				,	orig_sr=lowerCAmelCase__				,	target_sr=self.sampling_rate)
           __lowerCAmelCase	   =self.sampling_rate

      return {"path": path, "array": array, "sampling_rate": sampling_rate}



 def 		UpperCamelCase							(						self							:    Any)->							Union["FeatureType", Dict[str, "FeatureType"]]:
      from .features import Value

      if self.decode:
           raise ValueError("""Cannot flatten a decoded Audio feature.""")
      return {
          "bytes": Value("""binary"""),
          "path": Value("""string"""),
      }



 def 		UpperCamelCase							(						self							:    Dict				,	snake_case_							:    Union[pa.StringArray, pa.StructArray])->							pa.StructArray:
      if pa.types.is_string(storage.type):
           __lowerCAmelCase	   =pa.array([None] * len(lowerCAmelCase__)				,	type=pa.binary())
           __lowerCAmelCase	   =pa.StructArray.from_arrays([bytes_array, storage]				,	["""bytes""", """path"""]				,	mask=storage.is_null())
      elif pa.types.is_binary(storage.type):
           __lowerCAmelCase	   =pa.array([None] * len(lowerCAmelCase__)				,	type=pa.string())
           __lowerCAmelCase	   =pa.StructArray.from_arrays([storage, path_array]				,	["""bytes""", """path"""]				,	mask=storage.is_null())
      elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("""array"""):
           __lowerCAmelCase	   =pa.array([Audio().encode_example(lowerCAmelCase__) if x is not None else None for x in storage.to_pylist()])
      elif pa.types.is_struct(storage.type):
           if storage.type.get_field_index("""bytes""") >= 0:
                __lowerCAmelCase	   =storage.field("""bytes""")
           else:
                __lowerCAmelCase	   =pa.array([None] * len(lowerCAmelCase__)				,	type=pa.binary())
           if storage.type.get_field_index("""path""") >= 0:
                __lowerCAmelCase	   =storage.field("""path""")
           else:
                __lowerCAmelCase	   =pa.array([None] * len(lowerCAmelCase__)				,	type=pa.string())
           __lowerCAmelCase	   =pa.StructArray.from_arrays([bytes_array, path_array]				,	["""bytes""", """path"""]				,	mask=storage.is_null())
      return array_cast(lowerCAmelCase__				,	self.pa_type)



 def 		UpperCamelCase							(						self							:    Dict				,	snake_case_							:    pa.StructArray)->							pa.StructArray:

      @no_op_if_value_is_null
      def path_to_bytes(snake_case_							:    List[Any]):
           with xopen(lowerCAmelCase__				,	"""rb""") as f:
                __lowerCAmelCase	   =f.read()
           return bytes_

      __lowerCAmelCase	   =pa.array(
          [
              (path_to_bytes(x["""path"""]) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
              for x in storage.to_pylist()
          ]				,	type=pa.binary()				,	)
      __lowerCAmelCase	   =pa.array(
          [os.path.basename(lowerCAmelCase__) if path is not None else None for path in storage.field("""path""").to_pylist()]				,	type=pa.string()				,	)
      __lowerCAmelCase	   =pa.StructArray.from_arrays([bytes_array, path_array]				,	["""bytes""", """path"""]				,	mask=bytes_array.is_null())
      return array_cast(lowerCAmelCase__				,	self.pa_type)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_	       =			logging.get_logger(__name__)

lowercase_	       =			{
    '''google/realm-cc-news-pretrained-embedder''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json'''
    ),
    '''google/realm-cc-news-pretrained-encoder''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json'''
    ),
    '''google/realm-cc-news-pretrained-scorer''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json'''
    ),
    '''google/realm-cc-news-pretrained-openqa''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json'''
    ),
    '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json''',
    '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json''',
    '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json''',
    '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json''',
    # See all REALM models at https://huggingface.co/models?filter=realm
}






class 						__a (     SCREAMING_SNAKE_CASE				):
	SCREAMING_SNAKE_CASE       =       "realm"



	def __init__(						self							:    Tuple				,	snake_case_							:    Tuple=3_05_22				,	snake_case_							:    List[Any]=7_68				,	snake_case_							:    Any=1_28				,	snake_case_							:    Union[str, Any]=12				,	snake_case_							:    List[str]=12				,	snake_case_							:    Any=8				,	snake_case_							:    Any=30_72				,	snake_case_							:    int="gelu_new"				,	snake_case_							:    Optional[int]=0.1				,	snake_case_							:    Tuple=0.1				,	snake_case_							:    List[Any]=5_12				,	snake_case_							:    List[Any]=2				,	snake_case_							:    Any=0.0_2				,	snake_case_							:    List[str]=1e-12				,	snake_case_							:    Optional[Any]=2_56				,	snake_case_							:    Dict=10				,	snake_case_							:    str=1e-3				,	snake_case_							:    List[str]=5				,	snake_case_							:    Optional[Any]=3_20				,	snake_case_							:    Optional[Any]=13_35_37_18				,	snake_case_							:    Tuple=50_00				,	snake_case_							:    Union[str, Any]=1				,	snake_case_							:    Union[str, Any]=0				,	snake_case_							:    List[str]=2				,	**snake_case_							:    List[str]				,	)->							str:
						super().__init__(pad_token_id=snake_case_				,	bos_token_id=snake_case_				,	eos_token_id=snake_case_				,	**snake_case_)

						# Common config
						__lowerCAmelCase	   =vocab_size
						__lowerCAmelCase	   =max_position_embeddings
						__lowerCAmelCase	   =hidden_size
						__lowerCAmelCase	   =retriever_proj_size
						__lowerCAmelCase	   =num_hidden_layers
						__lowerCAmelCase	   =num_attention_heads
						__lowerCAmelCase	   =num_candidates
						__lowerCAmelCase	   =intermediate_size
						__lowerCAmelCase	   =hidden_act
						__lowerCAmelCase	   =hidden_dropout_prob
						__lowerCAmelCase	   =attention_probs_dropout_prob
						__lowerCAmelCase	   =initializer_range
						__lowerCAmelCase	   =type_vocab_size
						__lowerCAmelCase	   =layer_norm_eps

						# Reader config
						__lowerCAmelCase	   =span_hidden_size
						__lowerCAmelCase	   =max_span_width
						__lowerCAmelCase	   =reader_layer_norm_eps
						__lowerCAmelCase	   =reader_beam_size
						__lowerCAmelCase	   =reader_seq_len

						# Retrieval config
						__lowerCAmelCase	   =num_block_records
						__lowerCAmelCase	   =searcher_beam_size




