<jupyter_start><jupyter_text>Load Dataset<jupyter_code>from datasets import load_dataset

split = "train"
cache_dir = "./data_cache_doc2vec_solution_spans_notebook"

document_dataset = load_dataset(
    "doc2dial",
    name="document_domain",
    split=split,
    ignore_verifications=True,
    cache_dir=cache_dir,
)

rc_dataset = load_dataset(
    "doc2dial",
    name="doc2dial_rc",
    split=split,
    ignore_verifications=True,
    cache_dir=cache_dir,
)<jupyter_output><empty_output><jupyter_text>Train a model for each document  Preprocessing<jupyter_code>def span_dict_for_row(spans):
    spans_dict = {}
    for span in spans:
        spans_dict[span['id_sp']] = span['text_sp']

    return spans_dict

from gensim.utils import simple_preprocess
import pandas as pd

#Getting a list of spans per grounding document !!!! Language confusion: spans are the documents in gensim land and the list of spans (aka the whole document) is the corpora in gensim

document_full_df = pd.DataFrame(data=document_dataset)
raw_training_docs_per_doc = {}
for index, row in document_full_df.iterrows():
    raw_training_docs_per_doc[row['doc_id']] = span_dict_for_row(row['spans'])

tokenized_training_docs = {}
for doc_id in raw_training_docs_per_doc:
    spans_dic = raw_training_docs_per_doc[doc_id]
    preprocessed_spans = {}
    for key, value in spans_dic.items():
        #simplest preprocessing from gemsim
        preprocessed_spans[key] = simple_preprocess(value, deacc=True)
    tokenized_training_docs[doc_id] = preprocessed_spans

list(tokenized_training_docs.items())[0][1]<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>from gensim.models.doc2vec import Doc2Vec, TaggedDocument

models_for_doc = {}
for doc_id in tokenized_training_docs:
    tokenized_span_dic = tokenized_training_docs[doc_id]
    training_docs = [TaggedDocument(span, [key]) for key, span in tokenized_span_dic.items()]
    # Check how to fine tune the model https://radimrehurek.com/gensim/models/doc2vec.html
    model = Doc2Vec(training_docs, vector_size=10, window=4, min_count=1, workers=4, epochs=30)
    models_for_doc[doc_id] = model

len(models_for_doc)<jupyter_output><empty_output><jupyter_text>Predict rc questions<jupyter_code>predictions = []
references = []
for example in rc_dataset:
    question_ = example["question"]
    doc_id = example['title']
    # it does better if the user and agent string are left
    # question_ = question_.replace('user:', '')
    # question_ = question_.replace('agent:', '')

    #preprocess question in the same way
    test_doc = simple_preprocess(question_, deacc=True)
    #calculate vector using model for that document
    model = models_for_doc[doc_id]
    vector = model.infer_vector(test_doc)
    #find the most similar document (spans)
    sims = model.dv.most_similar([vector], topn=1)

    spans_dic = raw_training_docs_per_doc[doc_id]
    tag_for_most_likely_answer = sims[0][0]
    most_likely_answer = spans_dic[tag_for_most_likely_answer]

    # print(f'Question: {question_}\n')
    # print(f'Predicted Answer tag {tag_for_answer}: {most_likely_answer}\n')
    # print(f'Correct answer: {example["answers"]}\n')

    id_ = example["id"]
    predictions.append(
        {'id': id_,
         'prediction_text':
             most_likely_answer,
         'no_answer_probability': 0.0
         }
    )

    #just using their answers
    references.append(
        {
            "id": id_,
            "answers": example["answers"],
        }
    )

predictions[:5]<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>from datasets import load_metric

print(f'Number of question in train dataset {rc_dataset.shape[0]}')
metric = load_metric("squad_v2")
metric.add_batch(predictions=predictions, references=references)
final_score = metric.compute()
final_score<jupyter_output>Number of question in train dataset 20431<jupyter_text>How does this compare to guessing at random?<jupyter_code>import random

random_predictions = []
random_references = []
for example in rc_dataset:
    question_ = example["question"]
    doc_id = example['title']

    #pick a random text from the document
    spans = raw_training_docs_per_doc[doc_id]
    most_likely_answer = spans[str(random.randint(1, len(spans)))]

    id_ = example["id"]
    random_predictions.append(
        {'id': id_,
         'prediction_text':
             most_likely_answer,
         'no_answer_probability': 0.0
         }
    )

    #just using their answers
    random_references.append(
        {
            "id": id_,
            "answers": example["answers"],
        }
    )

random_predictions[:3]
print(f'Results for random text predictions for each question')
metric = load_metric("squad_v2")
metric.add_batch(predictions=random_predictions, references=random_references)
final_score = metric.compute()
final_score<jupyter_output>Results for random text predictions for each question<jupyter_text>Results for validation part of rc dataset<jupyter_code>rc_validation_dataset = load_dataset(
    "doc2dial",
    name="doc2dial_rc",
    split="validation",
    ignore_verifications=True,
    cache_dir=cache_dir,
)


validation_predictions = []
validation_references = []
print(f'Number of question in validation dataset {rc_validation_dataset.shape[0]}')

for example in rc_validation_dataset:
    question_ = example["question"]
    doc_id = example['title']
    # it does better if the user and agent string are left
    # question_ = question_.replace('user:', '')
    # question_ = question_.replace('agent:', '')

    #preprocess question in the same way
    test_doc = simple_preprocess(question_, deacc=True)
    #calculate vector using model for that document
    model = models_for_doc[doc_id]
    vector = model.infer_vector(test_doc)
    #find the most similar document (spans)
    sims = model.dv.most_similar([vector], topn=1)

    #find what that relates to in the original document
    spans_dic = raw_training_docs_per_doc[doc_id]
    tag_for_most_likely_answer = sims[0][0]
    most_likely_answer = spans_dic[tag_for_most_likely_answer]

    id_ = example["id"]
    validation_predictions.append(
        {'id': id_,
         'prediction_text':
             most_likely_answer,
         'no_answer_probability': 0.0
         }
    )

    #just using their answers
    validation_references.append(
        {
            "id": id_,
            "answers": example["answers"],
        }
    )

validation_predictions[:5]
print(f'Results for validation dataset')
metric = load_metric("squad_v2")
metric.add_batch(predictions=validation_predictions, references=validation_references)
final_score = metric.compute()
final_score<jupyter_output>Results for validation dataset
StarcoderdataJupyterStructuredCleanDedup