
'''simple docstring'''
import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
A__   :		int   =logging.getLogger()

A__   :		Union[str, Any]   =logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class  __A				( _SCREAMING_SNAKE_CASE			):
 def 		lowercase_(   self			:    int				,   lowerCamelCase			:    Optional[Any]       ):



   """simple docstring"""





   os.makedirs(lowerCamelCase				,   exist_ok=lowerCamelCase       )
   __A		:     Optional[Any]												=		{"""source""": """What is love ?""", """target""": """life"""}
   __A		:     List[str]												=		{"""train""": 12, """val""": 2, """test""": 2}
   for split in ["train", "test", "val"]:
     for field in ["source", "target"]:
       __A		:     List[Any]												=		"""\n""".join([contents[field]] * n_lines[split]       )
       with open(os.path.join(lowerCamelCase				,   f"{split}.{field}"       )				,   """w"""       ) as f:
         f.write(lowerCamelCase       )
 def 		lowercase_(   self			:    Optional[Any]				,   lowerCamelCase			:    int				,   lowerCamelCase			:    str = "pytorch"       ):



   """simple docstring"""





   __A		:     Optional[int]												=		self.get_auto_remove_tmp_dir()
   __A		:     Any												=		os.path.join(lowerCamelCase				,   """output"""       )
   __A		:     Union[str, Any]												=		os.path.join(lowerCamelCase				,   """data"""       )
   self._create_dummy_data(data_dir=lowerCamelCase       )

   __A		:     List[Any]												=		f"\n                --data_dir {data_dir} \\n                --output_dir {output_dir} \\n                --model_name_or_path facebook/rag-sequence-base \\n                --model_type rag_sequence \\n                --do_train \\n                --do_predict \\n                --n_val -1 \\n                --val_check_interval 1.0 \\n                --train_batch_size 2 \\n                --eval_batch_size 1 \\n                --max_source_length 25 \\n                --max_target_length 25 \\n                --val_max_target_length 25 \\n                --test_max_target_length 25 \\n                --label_smoothing 0.1 \\n                --dropout 0.1 \\n                --attention_dropout 0.1 \\n                --weight_decay 0.001 \\n                --adam_epsilon 1e-08 \\n                --max_grad_norm 0.1 \\n                --lr_scheduler polynomial \\n                --learning_rate 3e-04 \\n                --num_train_epochs 1 \\n                --warmup_steps 4 \\n                --gradient_accumulation_steps 1 \\n                --distributed-port 8787 \\n                --use_dummy_dataset 1 \\n                --distributed_retriever {distributed_retriever} \\n            ".split()

   if gpus > 0:
     testargs.append(f"--gpus={gpus}"       )
     if is_apex_available():
       testargs.append("""--fp16"""       )
   else:
     testargs.append("""--gpus=0"""       )
     testargs.append("""--distributed_backend=ddp_cpu"""       )
     testargs.append("""--num_processes=2"""       )

   __A		:     Tuple												=		[sys.executable, str(Path(finetune_rag.__file__       ).resolve()       )] + testargs
   execute_subprocess_async(lowerCamelCase				,   env=self.get_env()       )

   __A		:     int												=		os.path.join(lowerCamelCase				,   """metrics.json"""       )
   with open(lowerCamelCase       ) as f:
     __A		:     str												=		json.load(lowerCamelCase       )
   return result
 @require_torch_gpu
 def 		lowercase_(   self			:    Union[str, Any]       ):



   """simple docstring"""





   __A		:     Dict												=		self._run_finetune(gpus=1       )
   self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,   0.2       )
 @require_torch_multi_gpu
 def 		lowercase_(   self			:    int       ):



   """simple docstring"""





   __A		:     Tuple												=		self._run_finetune(gpus=2       )
   self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,   0.2       )
 @require_torch_gpu
 @require_ray
 def 		lowercase_(   self			:    List[Any]       ):



   """simple docstring"""





   __A		:     List[Any]												=		self._run_finetune(gpus=1				,   distributed_retriever="""ray"""       )
   self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,   0.2       )






 @require_torch_multi_gpu
 @require_ray
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     Optional[int]												=		self._run_finetune(gpus=1				,   distributed_retriever="""ray"""       )
   self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,   0.2       )





'''simple docstring'''
from typing import List, Optional, Union

import numpy as np

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


A__   :		Dict 		=logging.get_logger(__name__)







class  __A				( _SCREAMING_SNAKE_CASE			):
 lowerCamelCase				       =['''input_values''', '''padding_mask''']
 def __init__(   self			:    Optional[int]				,   lowerCamelCase			:    int = 1				,   lowerCamelCase			:    int = 2_40_00				,   lowerCamelCase			:    float = 0.0				,   lowerCamelCase			:    float = None				,   lowerCamelCase			:    float = None				,   **lowerCamelCase			:    Dict				,   ):



   """simple docstring"""





   super().__init__(feature_size=lowerCamelCase				,   sampling_rate=lowerCamelCase				,   padding_value=lowerCamelCase				,   **lowerCamelCase       )
   __A		:     str												=		chunk_length_s
   __A		:     str												=		overlap
 @property
 def 		lowercase_(   self			:    List[str]       ):



   """simple docstring"""





   if self.chunk_length_s is None:
     return None
   else:
     return int(self.chunk_length_s * self.sampling_rate       )
 @property
 def 		lowercase_(   self			:    List[str]       ):



   """simple docstring"""





   if self.chunk_length_s is None or self.overlap is None:
     return None
   else:
     return max(1				,   int((1.0 - self.overlap) * self.chunk_length       )       )






 def __call__(   self			:    List[str]				,   lowerCamelCase			:    Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]				,   lowerCamelCase			:    Optional[Union[bool, str, PaddingStrategy]] = None				,   lowerCamelCase			:    Optional[bool] = False				,   lowerCamelCase			:    Optional[int] = None				,   lowerCamelCase			:    Optional[Union[str, TensorType]] = None				,   lowerCamelCase			:    Optional[int] = None				,   ):



   """simple docstring"""





   if sampling_rate is not None:
     if sampling_rate != self.sampling_rate:
       raise ValueError(
           f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
           f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
           f" {self.sampling_rate} and not {sampling_rate}."       )
   else:
     logger.warning(
         """It is strongly recommended to pass the `sampling_rate` argument to this function. """
         """Failing to do so can result in silent errors that might be hard to debug."""       )

   if padding and truncation:
     raise ValueError("""Both padding and truncation were set. Make sure you only set one."""       )
   elif padding is None:
     # by default let's pad the inputs
     __A		:     List[Any]												=		True

   __A		:     List[str]												=		bool(
       isinstance(lowerCamelCase				,   (list, tuple)       ) and (isinstance(raw_audio[0]				,   (np.ndarray, tuple, list)       ))       )

   if is_batched:
     __A		:     Optional[Any]												=		[np.asarray(lowerCamelCase				,   dtype=np.floataa       ).T for audio in raw_audio]
   elif not is_batched and not isinstance(lowerCamelCase				,   np.ndarray       ):
     __A		:     Union[str, Any]												=		np.asarray(lowerCamelCase				,   dtype=np.floataa       )
   elif isinstance(lowerCamelCase				,   np.ndarray       ) and raw_audio.dtype is np.dtype(np.floataa       ):
     __A		:     Optional[Any]												=		raw_audio.astype(np.floataa       )

   # always return batch
   if not is_batched:
     __A		:     Tuple												=		[np.asarray(lowerCamelCase       ).T]

   # verify inputs are valid
   for idx, example in enumerate(lowerCamelCase       ):
     if example.ndim > 2:
       raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}"       )
     if self.feature_size == 1 and example.ndim != 1:
       raise ValueError(f"Expected mono audio but example has {example.shape[-1]} channels"       )
     if self.feature_size == 2 and example.shape[-1] != 2:
       raise ValueError(f"Expected stereo audio but example has {example.shape[-1]} channels"       )

   __A		:     Optional[Any]												=		None
   __A		:     Dict												=		BatchFeature({"""input_values""": raw_audio}       )
   if self.chunk_stride is not None and self.chunk_length is not None and max_length is None:
     if truncation:
       __A		:     Optional[Any]												=		min(array.shape[0] for array in raw_audio       )
       __A		:     Dict												=		int(np.floor(max_length / self.chunk_stride       )       )
       __A		:     Union[str, Any]												=		(nb_step - 1) * self.chunk_stride + self.chunk_length
     elif padding:
       __A		:     int												=		max(array.shape[0] for array in raw_audio       )
       __A		:     Optional[Any]												=		int(np.ceil(max_length / self.chunk_stride       )       )
       __A		:     Any												=		(nb_step - 1) * self.chunk_stride + self.chunk_length
       __A		:     Optional[int]												=		"""max_length"""
     else:
       __A		:     Optional[Any]												=		input_values

        # normal padding on batch
   if padded_inputs is None:
     __A		:     Tuple												=		self.pad(
         lowerCamelCase				,   max_length=lowerCamelCase				,   truncation=lowerCamelCase				,   padding=lowerCamelCase				,   return_attention_mask=lowerCamelCase				,   )
     if padding:
       __A		:     List[Any]												=		padded_inputs.pop("""attention_mask"""       )

   __A		:     List[Any]												=		[]
   for example in padded_inputs.pop("""input_values"""       ):
     if self.feature_size == 1:
       __A		:     Optional[Any]												=		example[..., None]
     input_values.append(example.T       )

   __A		:     str												=		input_values
   if return_tensors is not None:
     __A		:     Any												=		padded_inputs.convert_to_tensors(lowerCamelCase       )

   return padded_inputs



