We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
change naming of vars in tutorials
Tutorial 2
Change name of “train_data” variable to “data_dir”

Tutorial 5 in python and colab
Change parameter top_k_per_candidate into top_k


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `tutorials/Tutorial5_Evaluation.py`
Content:
```
1 from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
2 from haystack.preprocessor.utils import fetch_archive_from_http
3 from haystack.retriever.sparse import ElasticsearchRetriever
4 from haystack.retriever.dense import DensePassageRetriever
5 from haystack.eval import EvalAnswers, EvalDocuments
6 from haystack.reader.farm import FARMReader
7 from haystack.preprocessor import PreProcessor
8 from haystack.utils import launch_es
9 from haystack import Pipeline
10 
11 from farm.utils import initialize_device_settings
12 
13 import logging
14 
15 logger = logging.getLogger(__name__)
16 
17 
18 def tutorial5_evaluation():
19 
20     ##############################################
21     # Settings
22     ##############################################
23     # Choose from Evaluation style from ['retriever_closed', 'reader_closed', 'retriever_reader_open']
24     # 'retriever_closed' - evaluates only the retriever, based on whether the gold_label document is retrieved.
25     # 'reader_closed' - evaluates only the reader in a closed domain fashion i.e. the reader is given one query
26     #     and one document and metrics are calculated on whether the right position in this text is selected by
27     #     the model as the answer span (i.e. SQuAD style)
28     # 'retriever_reader_open' - evaluates retriever and reader in open domain fashion i.e. a document is considered
29     #     correctly retrieved if it contains the answer string within it. The reader is evaluated based purely on the
30     #     predicted string, regardless of which document this came from and the position of the extracted span.
31     style = "retriever_reader_open"
32 
33     # make sure these indices do not collide with existing ones, the indices will be wiped clean before data is inserted
34     doc_index = "tutorial5_docs"
35     label_index = "tutorial5_labels"
36 
37     ##############################################
38     # Code
39     ##############################################
40     launch_es()
41     device, n_gpu = initialize_device_settings(use_cuda=True)
42 
43     # Download evaluation data, which is a subset of Natural Questions development set containing 50 documents
44     doc_dir = "../data/nq"
45     s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset_v2.json.zip"
46     fetch_archive_from_http(url=s3_url, output_dir=doc_dir)
47 
48     # Connect to Elasticsearch
49     document_store = ElasticsearchDocumentStore(
50         host="localhost", username="", password="", index="document",
51         create_index=False, embedding_field="emb",
52         embedding_dim=768, excluded_meta_data=["emb"]
53     )
54 
55     # Add evaluation data to Elasticsearch document store
56     # We first delete the custom tutorial indices to not have duplicate elements
57     # and also split our documents into shorter passages using the PreProcessor
58     preprocessor = PreProcessor(
59         split_by="word",
60         split_length=500,
61         split_overlap=0,
62         split_respect_sentence_boundary=False,
63         clean_empty_lines=False,
64         clean_whitespace=False
65     )
66     document_store.delete_all_documents(index=doc_index)
67     document_store.delete_all_documents(index=label_index)
68     document_store.add_eval_data(
69         filename="../data/nq/nq_dev_subset_v2.json",
70         doc_index=doc_index,
71         label_index=label_index,
72         preprocessor=preprocessor
73     )
74 
75     # Let's prepare the labels that we need for the retriever and the reader
76     labels = document_store.get_all_labels_aggregated(index=label_index)
77 
78     # Initialize Retriever
79     retriever = ElasticsearchRetriever(document_store=document_store)
80 
81     # Alternative: Evaluate DensePassageRetriever
82     # Note, that DPR works best when you index short passages < 512 tokens as only those tokens will be used for the embedding.
83     # Here, for nq_dev_subset_v2.json we have avg. num of tokens = 5220(!).
84     # DPR still outperforms Elastic's BM25 by a small margin here.
85     # retriever = DensePassageRetriever(document_store=document_store,
86     #                                   query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
87     #                                   passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
88     #                                   use_gpu=True,
89     #                                   embed_title=True,
90     #                                   remove_sep_tok_from_untitled_passages=True)
91     # document_store.update_embeddings(retriever, index=doc_index)
92 
93     # Initialize Reader
94     reader = FARMReader(
95         model_name_or_path="deepset/roberta-base-squad2",
96         top_k_per_candidate=4,
97         return_no_answer=True
98     )
99 
100     # Here we initialize the nodes that perform evaluation
101     eval_retriever = EvalDocuments()
102     eval_reader = EvalAnswers()
103 
104 
105     ## Evaluate Retriever on its own in closed domain fashion
106     if style == "retriever_closed":
107         retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index)
108         ## Retriever Recall is the proportion of questions for which the correct document containing the answer is
109         ## among the correct documents
110         print("Retriever Recall:", retriever_eval_results["recall"])
111         ## Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
112         print("Retriever Mean Avg Precision:", retriever_eval_results["map"])
113 
114     # Evaluate Reader on its own in closed domain fashion (i.e. SQuAD style)
115     elif style == "reader_closed":
116         reader_eval_results = reader.eval(document_store=document_store, device=device, label_index=label_index, doc_index=doc_index)
117         # Evaluation of Reader can also be done directly on a SQuAD-formatted file without passing the data to Elasticsearch
118         #reader_eval_results = reader.eval_on_file("../data/nq", "nq_dev_subset_v2.json", device=device)
119 
120         ## Reader Top-N-Accuracy is the proportion of predicted answers that match with their corresponding correct answer
121         print("Reader Top-N-Accuracy:", reader_eval_results["top_n_accuracy"])
122         ## Reader Exact Match is the proportion of questions where the predicted answer is exactly the same as the correct answer
123         print("Reader Exact Match:", reader_eval_results["EM"])
124         ## Reader F1-Score is the average overlap between the predicted answers and the correct answers
125         print("Reader F1-Score:", reader_eval_results["f1"])
126 
127 
128     # Evaluate combination of Reader and Retriever in open domain fashion
129     elif style == "retriever_reader_open":
130 
131         # Here is the pipeline definition
132         p = Pipeline()
133         p.add_node(component=retriever, name="ESRetriever", inputs=["Query"])
134         p.add_node(component=eval_retriever, name="EvalDocuments", inputs=["ESRetriever"])
135         p.add_node(component=reader, name="QAReader", inputs=["EvalDocuments"])
136         p.add_node(component=eval_reader, name="EvalAnswers", inputs=["QAReader"])
137         results = []
138 
139         for l in labels:
140             res = p.run(
141                 query=l.question,
142                 top_k_retriever=10,
143                 labels=l,
144                 top_k_reader=10,
145                 index=doc_index,
146             )
147             results.append(res)
148 
149         eval_retriever.print()
150         print()
151         retriever.print_time()
152         print()
153         eval_reader.print(mode="reader")
154         print()
155         reader.print_time()
156         print()
157         eval_reader.print(mode="pipeline")
158     else:
159         raise ValueError(f'style={style} is not a valid option. Choose from retriever_closed, reader_closed, retriever_reader_open')
160 
161 
162 if __name__ == "__main__":
163     tutorial5_evaluation()
164 
165 # This Haystack script was made with love by deepset in Berlin, Germany
166 # Haystack: https://github.com/deepset-ai/haystack
167 # deepset: https://deepset.ai/
```

Path: `tutorials/Tutorial2_Finetune_a_model_on_your_data.py`
Content:
```
1 # # Fine-tuning a model on your own data
2 # 
3 # For many use cases it is sufficient to just use one of the existing public models that were trained on SQuAD or
4 # other public QA datasets (e.g. Natural Questions).
5 # However, if you have domain-specific questions, fine-tuning your model on custom examples will very likely boost
6 # your performance. While this varies by domain, we saw that ~ 2000 examples can easily increase performance by +5-20%.
7 # 
8 # This tutorial shows you how to fine-tune a pretrained model on your own dataset.
9 
10 from haystack.reader.farm import FARMReader
11 
12 
13 def tutorial2_finetune_a_model_on_your_data():
14     # ## Create Training Data
15     #
16     # There are two ways to generate training data
17     #
18     # 1. **Annotation**: You can use the annotation tool(https://github.com/deepset-ai/haystack#labeling-tool) to label
19     #                    your data, i.e. highlighting answers to your questions in a document. The tool supports structuring
20     #                   your workflow with organizations, projects, and users. The labels can be exported in SQuAD format
21     #                    that is compatible for training with Haystack.
22     #
23     # 2. **Feedback**:   For production systems, you can collect training data from direct user feedback via Haystack's
24     #                    REST API interface. This includes a customizable user feedback API for providing feedback on the
25     #                    answer returned by the API. The API provides a feedback export endpoint to obtain the feedback data
26     #                    for fine-tuning your model further.
27     #
28     #
29     # ## Fine-tune your model
30     #
31     # Once you have collected training data, you can fine-tune your base models.
32     # We initialize a reader as a base model and fine-tune it on our own custom dataset (should be in SQuAD-like format).
33     # We recommend using a base model that was trained on SQuAD or a similar QA dataset before to benefit from Transfer
34     # Learning effects.
35 
36     #**Recommendation: Run training on a GPU. To do so change the `use_gpu` arguments below to `True`
37 
38     reader = FARMReader(model_name_or_path="distilbert-base-uncased-distilled-squad", use_gpu=True)
39     train_data = "data/squad20"
40     # train_data = "PATH/TO_YOUR/TRAIN_DATA"
41     reader.train(data_dir=train_data, train_filename="dev-v2.0.json", use_gpu=True, n_epochs=1, save_dir="my_model")
42 
43     # Saving the model happens automatically at the end of training into the `save_dir` you specified
44     # However, you could also save a reader manually again via:
45     reader.save(directory="my_model")
46 
47     # If you want to load it at a later point, just do:
48     new_reader = FARMReader(model_name_or_path="my_model")
49 
50 
51 if __name__ == "__main__":
52     tutorial2_finetune_a_model_on_your_data()
53 
54 # This Haystack script was made with love by deepset in Berlin, Germany
55 # Haystack: https://github.com/deepset-ai/haystack
56 # deepset: https://deepset.ai/
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/tutorials/Tutorial2_Finetune_a_model_on_your_data.py b/tutorials/Tutorial2_Finetune_a_model_on_your_data.py
--- a/tutorials/Tutorial2_Finetune_a_model_on_your_data.py
+++ b/tutorials/Tutorial2_Finetune_a_model_on_your_data.py
@@ -36,9 +36,9 @@
     #**Recommendation: Run training on a GPU. To do so change the `use_gpu` arguments below to `True`
 
     reader = FARMReader(model_name_or_path="distilbert-base-uncased-distilled-squad", use_gpu=True)
-    train_data = "data/squad20"
-    # train_data = "PATH/TO_YOUR/TRAIN_DATA"
-    reader.train(data_dir=train_data, train_filename="dev-v2.0.json", use_gpu=True, n_epochs=1, save_dir="my_model")
+    data_dir = "data/squad20"
+    # data_dir = "PATH/TO_YOUR/TRAIN_DATA"
+    reader.train(data_dir=data_dir, train_filename="dev-v2.0.json", use_gpu=True, n_epochs=1, save_dir="my_model")
 
     # Saving the model happens automatically at the end of training into the `save_dir` you specified
     # However, you could also save a reader manually again via:
diff --git a/tutorials/Tutorial5_Evaluation.py b/tutorials/Tutorial5_Evaluation.py
--- a/tutorials/Tutorial5_Evaluation.py
+++ b/tutorials/Tutorial5_Evaluation.py
@@ -93,7 +93,7 @@
     # Initialize Reader
     reader = FARMReader(
         model_name_or_path="deepset/roberta-base-squad2",
-        top_k_per_candidate=4,
+        top_k=4,
         return_no_answer=True
     )

{"golden_diff": "diff --git a/tutorials/Tutorial2_Finetune_a_model_on_your_data.py b/tutorials/Tutorial2_Finetune_a_model_on_your_data.py\n--- a/tutorials/Tutorial2_Finetune_a_model_on_your_data.py\n+++ b/tutorials/Tutorial2_Finetune_a_model_on_your_data.py\n@@ -36,9 +36,9 @@\n     #**Recommendation: Run training on a GPU. To do so change the `use_gpu` arguments below to `True`\n \n     reader = FARMReader(model_name_or_path=\"distilbert-base-uncased-distilled-squad\", use_gpu=True)\n-    train_data = \"data/squad20\"\n-    # train_data = \"PATH/TO_YOUR/TRAIN_DATA\"\n-    reader.train(data_dir=train_data, train_filename=\"dev-v2.0.json\", use_gpu=True, n_epochs=1, save_dir=\"my_model\")\n+    data_dir = \"data/squad20\"\n+    # data_dir = \"PATH/TO_YOUR/TRAIN_DATA\"\n+    reader.train(data_dir=data_dir, train_filename=\"dev-v2.0.json\", use_gpu=True, n_epochs=1, save_dir=\"my_model\")\n \n     # Saving the model happens automatically at the end of training into the `save_dir` you specified\n     # However, you could also save a reader manually again via:\ndiff --git a/tutorials/Tutorial5_Evaluation.py b/tutorials/Tutorial5_Evaluation.py\n--- a/tutorials/Tutorial5_Evaluation.py\n+++ b/tutorials/Tutorial5_Evaluation.py\n@@ -93,7 +93,7 @@\n     # Initialize Reader\n     reader = FARMReader(\n         model_name_or_path=\"deepset/roberta-base-squad2\",\n-        top_k_per_candidate=4,\n+        top_k=4,\n         return_no_answer=True\n     )\n", "issue": "change naming of vars in tutorials\nTutorial 2\nChange name of \u201ctrain_data\u201d variable to \u201cdata_dir\u201d\n\nTutorial 5 in python and colab\nChange parameter top_k_per_candidate into top_k\n\n", "before_files": [{"content": "from haystack.document_store.elasticsearch import ElasticsearchDocumentStore\nfrom haystack.preprocessor.utils import fetch_archive_from_http\nfrom haystack.retriever.sparse import ElasticsearchRetriever\nfrom haystack.retriever.dense import DensePassageRetriever\nfrom haystack.eval import EvalAnswers, EvalDocuments\nfrom haystack.reader.farm import FARMReader\nfrom haystack.preprocessor import PreProcessor\nfrom haystack.utils import launch_es\nfrom haystack import Pipeline\n\nfrom farm.utils import initialize_device_settings\n\nimport logging\n\nlogger = logging.getLogger(__name__)\n\n\ndef tutorial5_evaluation():\n\n    ##############################################\n    # Settings\n    ##############################################\n    # Choose from Evaluation style from ['retriever_closed', 'reader_closed', 'retriever_reader_open']\n    # 'retriever_closed' - evaluates only the retriever, based on whether the gold_label document is retrieved.\n    # 'reader_closed' - evaluates only the reader in a closed domain fashion i.e. the reader is given one query\n    #     and one document and metrics are calculated on whether the right position in this text is selected by\n    #     the model as the answer span (i.e. SQuAD style)\n    # 'retriever_reader_open' - evaluates retriever and reader in open domain fashion i.e. a document is considered\n    #     correctly retrieved if it contains the answer string within it. The reader is evaluated based purely on the\n    #     predicted string, regardless of which document this came from and the position of the extracted span.\n    style = \"retriever_reader_open\"\n\n    # make sure these indices do not collide with existing ones, the indices will be wiped clean before data is inserted\n    doc_index = \"tutorial5_docs\"\n    label_index = \"tutorial5_labels\"\n\n    ##############################################\n    # Code\n    ##############################################\n    launch_es()\n    device, n_gpu = initialize_device_settings(use_cuda=True)\n\n    # Download evaluation data, which is a subset of Natural Questions development set containing 50 documents\n    doc_dir = \"../data/nq\"\n    s3_url = \"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset_v2.json.zip\"\n    fetch_archive_from_http(url=s3_url, output_dir=doc_dir)\n\n    # Connect to Elasticsearch\n    document_store = ElasticsearchDocumentStore(\n        host=\"localhost\", username=\"\", password=\"\", index=\"document\",\n        create_index=False, embedding_field=\"emb\",\n        embedding_dim=768, excluded_meta_data=[\"emb\"]\n    )\n\n    # Add evaluation data to Elasticsearch document store\n    # We first delete the custom tutorial indices to not have duplicate elements\n    # and also split our documents into shorter passages using the PreProcessor\n    preprocessor = PreProcessor(\n        split_by=\"word\",\n        split_length=500,\n        split_overlap=0,\n        split_respect_sentence_boundary=False,\n        clean_empty_lines=False,\n        clean_whitespace=False\n    )\n    document_store.delete_all_documents(index=doc_index)\n    document_store.delete_all_documents(index=label_index)\n    document_store.add_eval_data(\n        filename=\"../data/nq/nq_dev_subset_v2.json\",\n        doc_index=doc_index,\n        label_index=label_index,\n        preprocessor=preprocessor\n    )\n\n    # Let's prepare the labels that we need for the retriever and the reader\n    labels = document_store.get_all_labels_aggregated(index=label_index)\n\n    # Initialize Retriever\n    retriever = ElasticsearchRetriever(document_store=document_store)\n\n    # Alternative: Evaluate DensePassageRetriever\n    # Note, that DPR works best when you index short passages < 512 tokens as only those tokens will be used for the embedding.\n    # Here, for nq_dev_subset_v2.json we have avg. num of tokens = 5220(!).\n    # DPR still outperforms Elastic's BM25 by a small margin here.\n    # retriever = DensePassageRetriever(document_store=document_store,\n    #                                   query_embedding_model=\"facebook/dpr-question_encoder-single-nq-base\",\n    #                                   passage_embedding_model=\"facebook/dpr-ctx_encoder-single-nq-base\",\n    #                                   use_gpu=True,\n    #                                   embed_title=True,\n    #                                   remove_sep_tok_from_untitled_passages=True)\n    # document_store.update_embeddings(retriever, index=doc_index)\n\n    # Initialize Reader\n    reader = FARMReader(\n        model_name_or_path=\"deepset/roberta-base-squad2\",\n        top_k_per_candidate=4,\n        return_no_answer=True\n    )\n\n    # Here we initialize the nodes that perform evaluation\n    eval_retriever = EvalDocuments()\n    eval_reader = EvalAnswers()\n\n\n    ## Evaluate Retriever on its own in closed domain fashion\n    if style == \"retriever_closed\":\n        retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index)\n        ## Retriever Recall is the proportion of questions for which the correct document containing the answer is\n        ## among the correct documents\n        print(\"Retriever Recall:\", retriever_eval_results[\"recall\"])\n        ## Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank\n        print(\"Retriever Mean Avg Precision:\", retriever_eval_results[\"map\"])\n\n    # Evaluate Reader on its own in closed domain fashion (i.e. SQuAD style)\n    elif style == \"reader_closed\":\n        reader_eval_results = reader.eval(document_store=document_store, device=device, label_index=label_index, doc_index=doc_index)\n        # Evaluation of Reader can also be done directly on a SQuAD-formatted file without passing the data to Elasticsearch\n        #reader_eval_results = reader.eval_on_file(\"../data/nq\", \"nq_dev_subset_v2.json\", device=device)\n\n        ## Reader Top-N-Accuracy is the proportion of predicted answers that match with their corresponding correct answer\n        print(\"Reader Top-N-Accuracy:\", reader_eval_results[\"top_n_accuracy\"])\n        ## Reader Exact Match is the proportion of questions where the predicted answer is exactly the same as the correct answer\n        print(\"Reader Exact Match:\", reader_eval_results[\"EM\"])\n        ## Reader F1-Score is the average overlap between the predicted answers and the correct answers\n        print(\"Reader F1-Score:\", reader_eval_results[\"f1\"])\n\n\n    # Evaluate combination of Reader and Retriever in open domain fashion\n    elif style == \"retriever_reader_open\":\n\n        # Here is the pipeline definition\n        p = Pipeline()\n        p.add_node(component=retriever, name=\"ESRetriever\", inputs=[\"Query\"])\n        p.add_node(component=eval_retriever, name=\"EvalDocuments\", inputs=[\"ESRetriever\"])\n        p.add_node(component=reader, name=\"QAReader\", inputs=[\"EvalDocuments\"])\n        p.add_node(component=eval_reader, name=\"EvalAnswers\", inputs=[\"QAReader\"])\n        results = []\n\n        for l in labels:\n            res = p.run(\n                query=l.question,\n                top_k_retriever=10,\n                labels=l,\n                top_k_reader=10,\n                index=doc_index,\n            )\n            results.append(res)\n\n        eval_retriever.print()\n        print()\n        retriever.print_time()\n        print()\n        eval_reader.print(mode=\"reader\")\n        print()\n        reader.print_time()\n        print()\n        eval_reader.print(mode=\"pipeline\")\n    else:\n        raise ValueError(f'style={style} is not a valid option. Choose from retriever_closed, reader_closed, retriever_reader_open')\n\n\nif __name__ == \"__main__\":\n    tutorial5_evaluation()\n\n# This Haystack script was made with love by deepset in Berlin, Germany\n# Haystack: https://github.com/deepset-ai/haystack\n# deepset: https://deepset.ai/", "path": "tutorials/Tutorial5_Evaluation.py"}, {"content": "# # Fine-tuning a model on your own data\n# \n# For many use cases it is sufficient to just use one of the existing public models that were trained on SQuAD or\n# other public QA datasets (e.g. Natural Questions).\n# However, if you have domain-specific questions, fine-tuning your model on custom examples will very likely boost\n# your performance. While this varies by domain, we saw that ~ 2000 examples can easily increase performance by +5-20%.\n# \n# This tutorial shows you how to fine-tune a pretrained model on your own dataset.\n\nfrom haystack.reader.farm import FARMReader\n\n\ndef tutorial2_finetune_a_model_on_your_data():\n    # ## Create Training Data\n    #\n    # There are two ways to generate training data\n    #\n    # 1. **Annotation**: You can use the annotation tool(https://github.com/deepset-ai/haystack#labeling-tool) to label\n    #                    your data, i.e. highlighting answers to your questions in a document. The tool supports structuring\n    #                   your workflow with organizations, projects, and users. The labels can be exported in SQuAD format\n    #                    that is compatible for training with Haystack.\n    #\n    # 2. **Feedback**:   For production systems, you can collect training data from direct user feedback via Haystack's\n    #                    REST API interface. This includes a customizable user feedback API for providing feedback on the\n    #                    answer returned by the API. The API provides a feedback export endpoint to obtain the feedback data\n    #                    for fine-tuning your model further.\n    #\n    #\n    # ## Fine-tune your model\n    #\n    # Once you have collected training data, you can fine-tune your base models.\n    # We initialize a reader as a base model and fine-tune it on our own custom dataset (should be in SQuAD-like format).\n    # We recommend using a base model that was trained on SQuAD or a similar QA dataset before to benefit from Transfer\n    # Learning effects.\n\n    #**Recommendation: Run training on a GPU. To do so change the `use_gpu` arguments below to `True`\n\n    reader = FARMReader(model_name_or_path=\"distilbert-base-uncased-distilled-squad\", use_gpu=True)\n    train_data = \"data/squad20\"\n    # train_data = \"PATH/TO_YOUR/TRAIN_DATA\"\n    reader.train(data_dir=train_data, train_filename=\"dev-v2.0.json\", use_gpu=True, n_epochs=1, save_dir=\"my_model\")\n\n    # Saving the model happens automatically at the end of training into the `save_dir` you specified\n    # However, you could also save a reader manually again via:\n    reader.save(directory=\"my_model\")\n\n    # If you want to load it at a later point, just do:\n    new_reader = FARMReader(model_name_or_path=\"my_model\")\n\n\nif __name__ == \"__main__\":\n    tutorial2_finetune_a_model_on_your_data()\n\n# This Haystack script was made with love by deepset in Berlin, Germany\n# Haystack: https://github.com/deepset-ai/haystack\n# deepset: https://deepset.ai/", "path": "tutorials/Tutorial2_Finetune_a_model_on_your_data.py"}], "after_files": [{"content": "from haystack.document_store.elasticsearch import ElasticsearchDocumentStore\nfrom haystack.preprocessor.utils import fetch_archive_from_http\nfrom haystack.retriever.sparse import ElasticsearchRetriever\nfrom haystack.retriever.dense import DensePassageRetriever\nfrom haystack.eval import EvalAnswers, EvalDocuments\nfrom haystack.reader.farm import FARMReader\nfrom haystack.preprocessor import PreProcessor\nfrom haystack.utils import launch_es\nfrom haystack import Pipeline\n\nfrom farm.utils import initialize_device_settings\n\nimport logging\n\nlogger = logging.getLogger(__name__)\n\n\ndef tutorial5_evaluation():\n\n    ##############################################\n    # Settings\n    ##############################################\n    # Choose from Evaluation style from ['retriever_closed', 'reader_closed', 'retriever_reader_open']\n    # 'retriever_closed' - evaluates only the retriever, based on whether the gold_label document is retrieved.\n    # 'reader_closed' - evaluates only the reader in a closed domain fashion i.e. the reader is given one query\n    #     and one document and metrics are calculated on whether the right position in this text is selected by\n    #     the model as the answer span (i.e. SQuAD style)\n    # 'retriever_reader_open' - evaluates retriever and reader in open domain fashion i.e. a document is considered\n    #     correctly retrieved if it contains the answer string within it. The reader is evaluated based purely on the\n    #     predicted string, regardless of which document this came from and the position of the extracted span.\n    style = \"retriever_reader_open\"\n\n    # make sure these indices do not collide with existing ones, the indices will be wiped clean before data is inserted\n    doc_index = \"tutorial5_docs\"\n    label_index = \"tutorial5_labels\"\n\n    ##############################################\n    # Code\n    ##############################################\n    launch_es()\n    device, n_gpu = initialize_device_settings(use_cuda=True)\n\n    # Download evaluation data, which is a subset of Natural Questions development set containing 50 documents\n    doc_dir = \"../data/nq\"\n    s3_url = \"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset_v2.json.zip\"\n    fetch_archive_from_http(url=s3_url, output_dir=doc_dir)\n\n    # Connect to Elasticsearch\n    document_store = ElasticsearchDocumentStore(\n        host=\"localhost\", username=\"\", password=\"\", index=\"document\",\n        create_index=False, embedding_field=\"emb\",\n        embedding_dim=768, excluded_meta_data=[\"emb\"]\n    )\n\n    # Add evaluation data to Elasticsearch document store\n    # We first delete the custom tutorial indices to not have duplicate elements\n    # and also split our documents into shorter passages using the PreProcessor\n    preprocessor = PreProcessor(\n        split_by=\"word\",\n        split_length=500,\n        split_overlap=0,\n        split_respect_sentence_boundary=False,\n        clean_empty_lines=False,\n        clean_whitespace=False\n    )\n    document_store.delete_all_documents(index=doc_index)\n    document_store.delete_all_documents(index=label_index)\n    document_store.add_eval_data(\n        filename=\"../data/nq/nq_dev_subset_v2.json\",\n        doc_index=doc_index,\n        label_index=label_index,\n        preprocessor=preprocessor\n    )\n\n    # Let's prepare the labels that we need for the retriever and the reader\n    labels = document_store.get_all_labels_aggregated(index=label_index)\n\n    # Initialize Retriever\n    retriever = ElasticsearchRetriever(document_store=document_store)\n\n    # Alternative: Evaluate DensePassageRetriever\n    # Note, that DPR works best when you index short passages < 512 tokens as only those tokens will be used for the embedding.\n    # Here, for nq_dev_subset_v2.json we have avg. num of tokens = 5220(!).\n    # DPR still outperforms Elastic's BM25 by a small margin here.\n    # retriever = DensePassageRetriever(document_store=document_store,\n    #                                   query_embedding_model=\"facebook/dpr-question_encoder-single-nq-base\",\n    #                                   passage_embedding_model=\"facebook/dpr-ctx_encoder-single-nq-base\",\n    #                                   use_gpu=True,\n    #                                   embed_title=True,\n    #                                   remove_sep_tok_from_untitled_passages=True)\n    # document_store.update_embeddings(retriever, index=doc_index)\n\n    # Initialize Reader\n    reader = FARMReader(\n        model_name_or_path=\"deepset/roberta-base-squad2\",\n        top_k=4,\n        return_no_answer=True\n    )\n\n    # Here we initialize the nodes that perform evaluation\n    eval_retriever = EvalDocuments()\n    eval_reader = EvalAnswers()\n\n\n    ## Evaluate Retriever on its own in closed domain fashion\n    if style == \"retriever_closed\":\n        retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index)\n        ## Retriever Recall is the proportion of questions for which the correct document containing the answer is\n        ## among the correct documents\n        print(\"Retriever Recall:\", retriever_eval_results[\"recall\"])\n        ## Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank\n        print(\"Retriever Mean Avg Precision:\", retriever_eval_results[\"map\"])\n\n    # Evaluate Reader on its own in closed domain fashion (i.e. SQuAD style)\n    elif style == \"reader_closed\":\n        reader_eval_results = reader.eval(document_store=document_store, device=device, label_index=label_index, doc_index=doc_index)\n        # Evaluation of Reader can also be done directly on a SQuAD-formatted file without passing the data to Elasticsearch\n        #reader_eval_results = reader.eval_on_file(\"../data/nq\", \"nq_dev_subset_v2.json\", device=device)\n\n        ## Reader Top-N-Accuracy is the proportion of predicted answers that match with their corresponding correct answer\n        print(\"Reader Top-N-Accuracy:\", reader_eval_results[\"top_n_accuracy\"])\n        ## Reader Exact Match is the proportion of questions where the predicted answer is exactly the same as the correct answer\n        print(\"Reader Exact Match:\", reader_eval_results[\"EM\"])\n        ## Reader F1-Score is the average overlap between the predicted answers and the correct answers\n        print(\"Reader F1-Score:\", reader_eval_results[\"f1\"])\n\n\n    # Evaluate combination of Reader and Retriever in open domain fashion\n    elif style == \"retriever_reader_open\":\n\n        # Here is the pipeline definition\n        p = Pipeline()\n        p.add_node(component=retriever, name=\"ESRetriever\", inputs=[\"Query\"])\n        p.add_node(component=eval_retriever, name=\"EvalDocuments\", inputs=[\"ESRetriever\"])\n        p.add_node(component=reader, name=\"QAReader\", inputs=[\"EvalDocuments\"])\n        p.add_node(component=eval_reader, name=\"EvalAnswers\", inputs=[\"QAReader\"])\n        results = []\n\n        for l in labels:\n            res = p.run(\n                query=l.question,\n                top_k_retriever=10,\n                labels=l,\n                top_k_reader=10,\n                index=doc_index,\n            )\n            results.append(res)\n\n        eval_retriever.print()\n        print()\n        retriever.print_time()\n        print()\n        eval_reader.print(mode=\"reader\")\n        print()\n        reader.print_time()\n        print()\n        eval_reader.print(mode=\"pipeline\")\n    else:\n        raise ValueError(f'style={style} is not a valid option. Choose from retriever_closed, reader_closed, retriever_reader_open')\n\n\nif __name__ == \"__main__\":\n    tutorial5_evaluation()\n\n# This Haystack script was made with love by deepset in Berlin, Germany\n# Haystack: https://github.com/deepset-ai/haystack\n# deepset: https://deepset.ai/", "path": "tutorials/Tutorial5_Evaluation.py"}, {"content": "# # Fine-tuning a model on your own data\n# \n# For many use cases it is sufficient to just use one of the existing public models that were trained on SQuAD or\n# other public QA datasets (e.g. Natural Questions).\n# However, if you have domain-specific questions, fine-tuning your model on custom examples will very likely boost\n# your performance. While this varies by domain, we saw that ~ 2000 examples can easily increase performance by +5-20%.\n# \n# This tutorial shows you how to fine-tune a pretrained model on your own dataset.\n\nfrom haystack.reader.farm import FARMReader\n\n\ndef tutorial2_finetune_a_model_on_your_data():\n    # ## Create Training Data\n    #\n    # There are two ways to generate training data\n    #\n    # 1. **Annotation**: You can use the annotation tool(https://github.com/deepset-ai/haystack#labeling-tool) to label\n    #                    your data, i.e. highlighting answers to your questions in a document. The tool supports structuring\n    #                   your workflow with organizations, projects, and users. The labels can be exported in SQuAD format\n    #                    that is compatible for training with Haystack.\n    #\n    # 2. **Feedback**:   For production systems, you can collect training data from direct user feedback via Haystack's\n    #                    REST API interface. This includes a customizable user feedback API for providing feedback on the\n    #                    answer returned by the API. The API provides a feedback export endpoint to obtain the feedback data\n    #                    for fine-tuning your model further.\n    #\n    #\n    # ## Fine-tune your model\n    #\n    # Once you have collected training data, you can fine-tune your base models.\n    # We initialize a reader as a base model and fine-tune it on our own custom dataset (should be in SQuAD-like format).\n    # We recommend using a base model that was trained on SQuAD or a similar QA dataset before to benefit from Transfer\n    # Learning effects.\n\n    #**Recommendation: Run training on a GPU. To do so change the `use_gpu` arguments below to `True`\n\n    reader = FARMReader(model_name_or_path=\"distilbert-base-uncased-distilled-squad\", use_gpu=True)\n    data_dir = \"data/squad20\"\n    # data_dir = \"PATH/TO_YOUR/TRAIN_DATA\"\n    reader.train(data_dir=data_dir, train_filename=\"dev-v2.0.json\", use_gpu=True, n_epochs=1, save_dir=\"my_model\")\n\n    # Saving the model happens automatically at the end of training into the `save_dir` you specified\n    # However, you could also save a reader manually again via:\n    reader.save(directory=\"my_model\")\n\n    # If you want to load it at a later point, just do:\n    new_reader = FARMReader(model_name_or_path=\"my_model\")\n\n\nif __name__ == \"__main__\":\n    tutorial2_finetune_a_model_on_your_data()\n\n# This Haystack script was made with love by deepset in Berlin, Germany\n# Haystack: https://github.com/deepset-ai/haystack\n# deepset: https://deepset.ai/", "path": "tutorials/Tutorial2_Finetune_a_model_on_your_data.py"}]}