

import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


lowerCAmelCase__       = pytest.mark.integration






@require_faiss
class 						__magic_name__    (       __lowercase    ):


 def _UpperCamelCase	(						self		: Dict    )    ->      int:
   UpperCAmelCase					      =  Dataset.from_dict({"filename": ["my_name-train" + "_" + str(lowerCAmelCase__    ) for x in np.arange(3_0    ).tolist()]}    )
   return dset


 def _UpperCamelCase	(						self		: Tuple    )    ->      Any:
   import faiss

   UpperCAmelCase					      =  self._create_dummy_dataset()
   UpperCAmelCase					      =  dset.map(
       lambda lowerCAmelCase__			,						lowerCAmelCase__    : {"vecs": i * np.ones(5			,						dtype=np.floataa    )}			,						with_indices=lowerCAmelCase__			,						keep_in_memory=lowerCAmelCase__    )
   UpperCAmelCase					      =  dset.add_faiss_index("vecs"			,						batch_size=1_0_0			,						metric_type=faiss.METRIC_INNER_PRODUCT    )
   UpperCAmelCase					      =  dset.get_nearest_examples("vecs"			,						np.ones(5			,						dtype=np.floataa    )    )
   self.assertEqual(examples["filename"][0]			,						"my_name-train_29"    )
   dset.drop_index("vecs"    )


 def _UpperCamelCase	(						self		: int    )    ->      List[str]:
   import faiss

   UpperCAmelCase					      =  self._create_dummy_dataset()
   dset.add_faiss_index_from_external_arrays(
       external_arrays=np.ones((3_0, 5)    ) * np.arange(3_0    ).reshape(-1			,						1    )			,						index_name="vecs"			,						batch_size=1_0_0			,						metric_type=faiss.METRIC_INNER_PRODUCT			,						)
   UpperCAmelCase					      =  dset.get_nearest_examples("vecs"			,						np.ones(5			,						dtype=np.floataa    )    )
   self.assertEqual(examples["filename"][0]			,						"my_name-train_29"    )


 def _UpperCamelCase	(						self		: List[Any]    )    ->      Union[str, Any]:
   import faiss

   UpperCAmelCase					      =  self._create_dummy_dataset()
   dset.add_faiss_index_from_external_arrays(
       external_arrays=np.ones((3_0, 5)    ) * np.arange(3_0    ).reshape(-1			,						1    )			,						index_name="vecs"			,						metric_type=faiss.METRIC_INNER_PRODUCT			,						)

   # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
   # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
   # see https://bugs.python.org/issue14243 and
   # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
   with tempfile.NamedTemporaryFile(delete=lowerCAmelCase__    ) as tmp_file:
     dset.save_faiss_index("vecs"			,						tmp_file.name    )
     dset.load_faiss_index("vecs2"			,						tmp_file.name    )
   os.unlink(tmp_file.name    )

   UpperCAmelCase					      =  dset.get_nearest_examples("vecs2"			,						np.ones(5			,						dtype=np.floataa    )    )
   self.assertEqual(examples["filename"][0]			,						"my_name-train_29"    )


 def _UpperCamelCase	(						self		: Union[str, Any]    )    ->      Optional[int]:
   UpperCAmelCase					      =  self._create_dummy_dataset()
   dset.add_faiss_index_from_external_arrays(
       external_arrays=np.ones((3_0, 5)    ) * np.arange(3_0    ).reshape(-1			,						1    )			,						index_name="vecs"    )
   dset.drop_index("vecs"    )
   self.assertRaises(lowerCAmelCase__			,						partial(dset.get_nearest_examples			,						"vecs2"			,						np.ones(5			,						dtype=np.floataa    )    )    )


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      int:
   from elasticsearch import Elasticsearch

   UpperCAmelCase					      =  self._create_dummy_dataset()
   with patch("elasticsearch.Elasticsearch.search"    ) as mocked_search, patch(
       "elasticsearch.client.IndicesClient.create"    ) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"    ) as mocked_bulk:
     UpperCAmelCase					      =  {"acknowledged": True}
     mocked_bulk.return_value([(True, None)] * 3_0    )
     UpperCAmelCase					      =  {"hits": {"hits": [{"_score": 1, "_id": 2_9}]}}
     UpperCAmelCase					      =  Elasticsearch()

     dset.add_elasticsearch_index("filename"			,						es_client=lowerCAmelCase__    )
     UpperCAmelCase					      =  dset.get_nearest_examples("filename"			,						"my_name-train_29"    )
     self.assertEqual(examples["filename"][0]			,						"my_name-train_29"    )






@require_faiss
class 						__magic_name__    (       __lowercase    ):


 def _UpperCamelCase	(						self		: Optional[int]    )    ->      str:
   import faiss

   UpperCAmelCase					      =  FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT    )

   # add vectors
   index.add_vectors(np.eye(5			,						dtype=np.floataa    )    )
   self.assertIsNotNone(index.faiss_index    )
   self.assertEqual(index.faiss_index.ntotal			,						5    )
   index.add_vectors(np.zeros((5, 5)			,						dtype=np.floataa    )    )
   self.assertEqual(index.faiss_index.ntotal			,						1_0    )

   # single query
   UpperCAmelCase					      =  np.zeros(5			,						dtype=np.floataa    )
   UpperCAmelCase					      =  1
   UpperCAmelCase					      =  index.search(lowerCAmelCase__    )
   self.assertRaises(lowerCAmelCase__			,						index.search			,						query.reshape(-1			,						1    )    )
   self.assertGreater(scores[0]			,						0    )
   self.assertEqual(indices[0]			,						1    )

   # batched queries
   UpperCAmelCase					      =  np.eye(5			,						dtype=np.floataa    )[::-1]
   UpperCAmelCase					      =  index.search_batch(lowerCAmelCase__    )
   self.assertRaises(lowerCAmelCase__			,						index.search_batch			,						queries[0]    )
   UpperCAmelCase					      =  [scores[0] for scores in total_scores]
   UpperCAmelCase					      =  [indices[0] for indices in total_indices]
   self.assertGreater(np.min(lowerCAmelCase__    )			,						0    )
   self.assertListEqual([4, 3, 2, 1, 0]			,						lowerCAmelCase__    )


 def _UpperCamelCase	(						self		: List[Any]    )    ->      Any:
   import faiss

   UpperCAmelCase					      =  FaissIndex(string_factory="Flat"    )
   index.add_vectors(np.eye(5			,						dtype=np.floataa    )    )
   self.assertIsInstance(index.faiss_index			,						faiss.IndexFlat    )
   UpperCAmelCase					      =  FaissIndex(string_factory="LSH"    )
   index.add_vectors(np.eye(5			,						dtype=np.floataa    )    )
   self.assertIsInstance(index.faiss_index			,						faiss.IndexLSH    )
   with self.assertRaises(lowerCAmelCase__    ):
     UpperCAmelCase					      =  FaissIndex(string_factory="Flat"			,						custom_index=faiss.IndexFlat(5    )    )


 def _UpperCamelCase	(						self		: List[str]    )    ->      Optional[Any]:
   import faiss

   UpperCAmelCase					      =  faiss.IndexFlat(5    )
   UpperCAmelCase					      =  FaissIndex(custom_index=lowerCAmelCase__    )
   index.add_vectors(np.eye(5			,						dtype=np.floataa    )    )
   self.assertIsInstance(index.faiss_index			,						faiss.IndexFlat    )


 def _UpperCamelCase	(						self		: Any    )    ->      Any:
   import faiss

   UpperCAmelCase					      =  FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT    )
   index.add_vectors(np.eye(5			,						dtype=np.floataa    )    )

   # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
   # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
   # see https://bugs.python.org/issue14243 and
   # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
   with tempfile.NamedTemporaryFile(delete=lowerCAmelCase__    ) as tmp_file:
     index.save(tmp_file.name    )
     UpperCAmelCase					      =  FaissIndex.load(tmp_file.name    )
   os.unlink(tmp_file.name    )

   UpperCAmelCase					      =  np.zeros(5			,						dtype=np.floataa    )
   UpperCAmelCase					      =  1
   UpperCAmelCase					      =  index.search(lowerCAmelCase__    )
   self.assertGreater(scores[0]			,						0    )
   self.assertEqual(indices[0]			,						1    )

@require_faiss
def      _lowerCAmelCase(	__A			):
  import faiss

  UpperCAmelCase					      =  FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)
  index.add_vectors(np.eye(5   ,       dtype=np.floataa			)			)

  UpperCAmelCase					      =  "index.faiss"
  UpperCAmelCase					      =  F"mock://{index_name}"
  index.save(_lowerCamelCase   ,       storage_options=mockfs.storage_options			)
  UpperCAmelCase					      =  FaissIndex.load(_lowerCamelCase   ,       storage_options=mockfs.storage_options			)

  UpperCAmelCase					      =  np.zeros(5   ,       dtype=np.floataa			)
  UpperCAmelCase					      =  1
  UpperCAmelCase					      =  index.search(_lowerCamelCase			)
  assert scores[0] > 0
  assert indices[0] == 1






@require_elasticsearch
class 						__magic_name__    (       __lowercase    ):


 def _UpperCamelCase	(						self		: List[str]    )    ->      str:
   from elasticsearch import Elasticsearch

   with patch("elasticsearch.Elasticsearch.search"    ) as mocked_search, patch(
       "elasticsearch.client.IndicesClient.create"    ) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"    ) as mocked_bulk:
     UpperCAmelCase					      =  Elasticsearch()
     UpperCAmelCase					      =  {"acknowledged": True}
     UpperCAmelCase					      =  ElasticSearchIndex(es_client=lowerCAmelCase__    )
     mocked_bulk.return_value([(True, None)] * 3    )
     index.add_documents(["foo", "bar", "foobar"]    )

     # single query
     UpperCAmelCase					      =  "foo"
     UpperCAmelCase					      =  {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
     UpperCAmelCase					      =  index.search(lowerCAmelCase__    )
     self.assertEqual(scores[0]			,						1    )
     self.assertEqual(indices[0]			,						0    )

     # single query with timeout
     UpperCAmelCase					      =  "foo"
     UpperCAmelCase					      =  {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
     UpperCAmelCase					      =  index.search(lowerCAmelCase__			,						request_timeout=3_0    )
     self.assertEqual(scores[0]			,						1    )
     self.assertEqual(indices[0]			,						0    )

     # batched queries
     UpperCAmelCase					      =  ["foo", "bar", "foobar"]
     UpperCAmelCase					      =  {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
     UpperCAmelCase					      =  index.search_batch(lowerCAmelCase__    )
     UpperCAmelCase					      =  [scores[0] for scores in total_scores]
     UpperCAmelCase					      =  [indices[0] for indices in total_indices]
     self.assertGreater(np.min(lowerCAmelCase__    )			,						0    )
     self.assertListEqual([1, 1, 1]			,						lowerCAmelCase__    )

     # batched queries with timeout
     UpperCAmelCase					      =  ["foo", "bar", "foobar"]
     UpperCAmelCase					      =  {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
     UpperCAmelCase					      =  index.search_batch(lowerCAmelCase__			,						request_timeout=3_0    )
     UpperCAmelCase					      =  [scores[0] for scores in total_scores]
     UpperCAmelCase					      =  [indices[0] for indices in total_indices]
     self.assertGreater(np.min(lowerCAmelCase__    )			,						0    )
     self.assertListEqual([1, 1, 1]			,						lowerCAmelCase__    )






from . import (
    albert,
    align,
    altclip,
    audio_spectrogram_transformer,
    auto,
    autoformer,
    bark,
    bart,
    barthez,
    bartpho,
    beit,
    bert,
    bert_generation,
    bert_japanese,
    bertweet,
    big_bird,
    bigbird_pegasus,
    biogpt,
    bit,
    blenderbot,
    blenderbot_small,
    blip,
    blip_a,
    bloom,
    bridgetower,
    byta,
    camembert,
    canine,
    chinese_clip,
    clap,
    clip,
    clipseg,
    codegen,
    conditional_detr,
    convbert,
    convnext,
    convnextva,
    cpm,
    cpmant,
    ctrl,
    cvt,
    dataavec,
    deberta,
    deberta_va,
    decision_transformer,
    deformable_detr,
    deit,
    deprecated,
    deta,
    detr,
    dialogpt,
    dinat,
    distilbert,
    dit,
    donut,
    dpr,
    dpt,
    efficientformer,
    efficientnet,
    electra,
    encodec,
    encoder_decoder,
    ernie,
    ernie_m,
    esm,
    falcon,
    flaubert,
    flava,
    fnet,
    focalnet,
    fsmt,
    funnel,
    git,
    glpn,
    gpta,
    gpt_bigcode,
    gpt_neo,
    gpt_neox,
    gpt_neox_japanese,
    gpt_swa,
    gptj,
    gptsan_japanese,
    graphormer,
    groupvit,
    herbert,
    hubert,
    ibert,
    imagegpt,
    informer,
    instructblip,
    jukebox,
    layoutlm,
    layoutlmva,
    layoutlmva,
    layoutxlm,
    led,
    levit,
    lilt,
    llama,
    longformer,
    longta,
    luke,
    lxmert,
    mam_aaa,
    marian,
    markuplm,
    maskaformer,
    maskformer,
    mbart,
    mbartaa,
    mega,
    megatron_bert,
    megatron_gpta,
    mgp_str,
    mluke,
    mobilebert,
    mobilenet_va,
    mobilenet_va,
    mobilevit,
    mobilevitva,
    mpnet,
    mra,
    mta,
    musicgen,
    mvp,
    nat,
    nezha,
    nllb,
    nllb_moe,
    nystromformer,
    oneformer,
    open_llama,
    openai,
    opt,
    owlvit,
    pegasus,
    pegasus_x,
    perceiver,
    phobert,
    pixastruct,
    plbart,
    poolformer,
    prophetnet,
    qdqbert,
    rag,
    realm,
    reformer,
    regnet,
    rembert,
    resnet,
    roberta,
    roberta_prelayernorm,
    roc_bert,
    roformer,
    rwkv,
    sam,
    segformer,
    sew,
    sew_d,
    speech_encoder_decoder,
    speech_to_text,
    speech_to_text_a,
    speechta,
    splinter,
    squeezebert,
    swiftformer,
    swin,
    swinasr,
    swinva,
    switch_transformers,
    ta,
    table_transformer,
    tapas,
    time_series_transformer,
    timesformer,
    timm_backbone,
    transfo_xl,
    trocr,
    tvlt,
    umta,
    unispeech,
    unispeech_sat,
    upernet,
    videomae,
    vilt,
    vision_encoder_decoder,
    vision_text_dual_encoder,
    visual_bert,
    vit,
    vit_hybrid,
    vit_mae,
    vit_msn,
    vivit,
    wavaveca,
    wavaveca_conformer,
    wavaveca_phoneme,
    wavaveca_with_lm,
    wavlm,
    whisper,
    x_clip,
    xglm,
    xlm,
    xlm_prophetnet,
    xlm_roberta,
    xlm_roberta_xl,
    xlnet,
    xmod,
    yolos,
    yoso,
)



