# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: env
#     language: python
#     name: env
# ---

# make sure these indices do not collide with existing ones, the indices will be wiped clean before data is inserted
doc_index = "squad_docs"
label_index = "squad_labels"

# # 200 split_lenght
#
# Estos eran los parametros por defecto encontrados en los tutoriales de haystack

# +
model_name = ['model_1', 'model_2', 'model_3', 'model_4', 'model_5', 'model_6', 'model_7', 'model_8', 'model_9'
             ,'model_10', 'model_1', 'base_model', 'old_base_model', 'bm25']

test_200_map = []
test_200_recall = []
dev_200_map = []
dev_200_recall = []

# +
# Connect to Elasticsearch
# docker start es01-test -a
from haystack.document_stores import ElasticsearchDocumentStore

document_store = ElasticsearchDocumentStore(
    host="localhost",
    username="",
    password="",
    index=doc_index,
    label_index=label_index,
    embedding_field="emb",
    embedding_dim=768,
    excluded_meta_data=["emb"],
)

from haystack.nodes import PreProcessor

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

preprocessor = PreProcessor(
    split_length=200,
    split_by="word",
    split_overlap=0,
    split_respect_sentence_boundary=False,
    clean_empty_lines=False,
    clean_whitespace=False
)
# -

# ## Model 1

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_1"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_1"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 2

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_2"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_2"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 3

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_3"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_3"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 4

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_4"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_4"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 5

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_5"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_5"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 6

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_6"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_6"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 7

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_7"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_7"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 8

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_8"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_8"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 9

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_9"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_9"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Model 10

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_10"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_10"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Base Model

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test Set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="IIC/dpr-spanish-question_encoder-allqa-base",
            passage_embedding_model="IIC/dpr-spanish-passage_encoder-allqa-base",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev Set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="IIC/dpr-spanish-question_encoder-allqa-base",
            passage_embedding_model="IIC/dpr-spanish-passage_encoder-allqa-base",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## Old Base Model

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test Set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="voidful/dpr-question_encoder-bert-base-multilingual",
            passage_embedding_model="voidful/dpr-ctx_encoder-bert-base-multilingual",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev Set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="voidful/dpr-question_encoder-bert-base-multilingual",
            passage_embedding_model="voidful/dpr-ctx_encoder-bert-base-multilingual",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# ## BM25

# +
# Initialize Retriever
from haystack.nodes import ElasticsearchRetriever, BM25Retriever

retriever = BM25Retriever(document_store=document_store)
# -

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test Set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor
)

## Evaluate Retriever on its own #THIS IS THE ES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_200_map.append(retriever_eval_results["map"])

# ### Dev Set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor
)

## Evaluate Retriever on its own #THIS IS THE ES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_200_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_200_map.append(retriever_eval_results["map"])

# # 400 split_lenght
#
# Estos eran los parametros por defecto en el area de entrenamiento de haystack, tambien son los mismos parametros con que se subieron los documentos

# +
model_name = ['model_1', 'model_2', 'model_3', 'model_4', 'model_5', 'model_6', 'model_7', 'model_8', 'model_9'
             ,'model_10', 'base_model', 'old_base_model', 'bm25']

test_400_map = []
test_400_recall = []
dev_400_map = []
dev_400_recall = []

# +
# Connect to Elasticsearch
# docker start es01-test -a
from haystack.document_stores import ElasticsearchDocumentStore

document_store = ElasticsearchDocumentStore(
    host="localhost",
    username="",
    password="",
    index=doc_index,
    label_index=label_index,
    embedding_field="emb",
    embedding_dim=768,
    excluded_meta_data=["emb"],
)

from haystack.nodes import PreProcessor

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

preprocessor = PreProcessor(
    clean_empty_lines=False, #Not supported
    clean_whitespace=False, #Not supported
    split_by="word",
    split_length=400,
    split_respect_sentence_boundary=False, #Not supported
    split_overlap=0,
    language="es"
)

# -

# ## Model 1

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_1"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_1"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 2

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_2"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_2"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 3

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_3"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_3"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 4

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_4"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_4"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 5

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_5"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_5"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 6

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_6"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_6"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 7

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_7"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_7"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 8

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_8"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_8"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 9

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_9"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_9"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Model 10

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_10"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

# +
from haystack.nodes import DensePassageRetriever

save_dir = "models/model_10"

retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=document_store, use_gpu=True)
# -

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own # THIS IS THE FINE-TUNED model_1
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Base Model

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test Set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="IIC/dpr-spanish-question_encoder-allqa-base",
            passage_embedding_model="IIC/dpr-spanish-passage_encoder-allqa-base",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev Set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="IIC/dpr-spanish-question_encoder-allqa-base",
            passage_embedding_model="IIC/dpr-spanish-passage_encoder-allqa-base",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## Old Base Model

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test Set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="voidful/dpr-question_encoder-bert-base-multilingual",
            passage_embedding_model="voidful/dpr-ctx_encoder-bert-base-multilingual",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev Set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor,
)

from haystack.nodes import DensePassageRetriever
retriever = DensePassageRetriever(
            document_store=document_store,
            query_embedding_model="voidful/dpr-question_encoder-bert-base-multilingual",
            passage_embedding_model="voidful/dpr-ctx_encoder-bert-base-multilingual",
            use_gpu=True,
            batch_size = 64
            )

document_store.update_embeddings(retriever, index=doc_index)

## Evaluate Retriever on its own #THIS IS THE BARE BONES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# ## BM25

# +
# Initialize Retriever
from haystack.nodes import ElasticsearchRetriever, BM25Retriever

retriever = BM25Retriever(document_store=document_store)
# -

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# ### Test Set

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/test.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor
)

## Evaluate Retriever on its own #THIS IS THE ES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
test_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
test_400_map.append(retriever_eval_results["map"])

# ### Dev Set

document_store.delete_documents(index=doc_index)
document_store.delete_documents(index=label_index)

# The add_eval_data() method converts the given dataset in json format into Haystack document and label objects. Those objects are then indexed in their respective document and label index in the document store. The method can be used with any dataset in SQuAD format.
document_store.add_eval_data(
    filename="squad_format_thesis/dev.json",
    doc_index=doc_index,
    label_index=label_index,
    preprocessor=preprocessor
)

## Evaluate Retriever on its own #THIS IS THE ES
# Note that no_answer samples are omitted when evaluation is performed with this method
retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index, open_domain=True)
# Retriever Recall is the proportion of questions for which the correct document containing the answer is
# among the correct documents
print("Retriever Recall:", retriever_eval_results["recall"])
dev_400_recall.append(retriever_eval_results["recall"])
# Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
dev_400_map.append(retriever_eval_results["map"])

# # Build the data

# +
print(len(model_name))
print("200")
print("TEST_RECALL", len(test_200_recall), "\n", [round(x,3) for x in test_200_recall])
print("TEST_MAP", len(test_200_map), "\n", [round(x,3) for x in test_200_map])
print("DEV_RECALL", len(dev_200_recall), "\n",  [round(x,3) for x in dev_200_recall])
print("DEV_MAP", len(dev_200_map),"\n",  [round(x,3) for x in dev_200_map])
print("400")
print("TEST_RECALL\n", len(test_400_recall) , "\n", [round(x,3) for x in test_400_recall])
print("TEST_MAP\n", len(test_400_map), "\n", [round(x,3) for x in test_400_map])
print("DEV_RECALL\n", len(dev_400_recall), "\n", [round(x,3) for x in dev_400_recall])
print("DEV_MAP", len(dev_400_map), "\n", [round(x,3) for x in dev_400_map])

r_test_200_recall = [round(x,3) for x in test_200_recall]
r_test_200_map = [round(x,3) for x in test_200_map]
r_dev_200_recall = [round(x,3) for x in dev_200_recall]
r_dev_200_map = [round(x,3) for x in dev_200_map]

r_test_400_recall = [round(x,3) for x in test_400_recall]
r_test_400_map = [round(x,3) for x in test_400_map]
r_dev_400_recall = [round(x,3) for x in dev_400_recall]
r_dev_400_map = [round(x,3) for x in dev_400_map]
# -

data_eval = {"model_name" : model_name, "test_200_recall": r_test_200_recall, "test_200_map": r_test_200_map,
            "dev_200_recall": r_dev_200_recall, "dev_200_map": r_dev_200_map, "test_400_recall": r_test_400_recall,
            "test_400_map": r_test_400_map, "dev_400_recall": r_dev_400_recall, "dev_400_map": r_dev_400_map}
data_eval

# +
import pandas as pd

df_data_eval = pd.DataFrame(data_eval)
df_data_eval
# -

df_data_eval.to_csv("data/df_data_eval.csv", index=False)



StarcoderdataJupyterScriptsDedupFiltered