# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: LiveProject
#     language: python
#     name: liveproject
# ---

# # Milestone 4

# Set up the questions and the paragraphs 

# +
import pandas as pd

df = pd.read_csv("paragraphs.csv")

questions = [
    ["Does a company need to keep track of the carbon intensity of the electricity?"],
    ["What metric is used for evaluating emission?"],
    ["How does one get to net-zero emissions economy?"],
    ["What is net-zero emissions economy?"],
    ["How can carbon emission of the processes of cement clinker be reduced?"],
    ["How is the Weighted Cogeneration Threshold calculated?"],
    ["What is carbon capture and sequestration?"],
    ["What stages does CCS consist of?"],
    ["What should be the average energy consumption of a water supply system?"],
    ["What are sludge treatments?"],
    ["How is the process of anaerobic digestion?"],
    ["What is considered Zero direct emission vehicles?"]
]

# -

# Set up the vectorizer and user a function to most similar paragraph to the question.

# +
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel

vectorizer = TfidfVectorizer()
vector_corpus = vectorizer.fit_transform(df["paragraph"])


def get_context(question):
    q_v = vectorizer.transform(question)
    lk_rank = linear_kernel(q_v, vector_corpus).flatten()
    return df["paragraph"][lk_rank.argsort()[-1]]


# -

# Initiate the QA pipeline and a function which return the answer

# +
from transformers import pipeline


MODEL = "distilbert-base-uncased-distilled-squad"
qamodel = pipeline("question-answering", model=MODEL, tokenizer=MODEL, device=-1)

def get_answer_pipeline(question, context):
    answer = qamodel(question=question, context=context)
    return answer["answer"].rstrip(".").rstrip(",").lstrip("(").rstrip(")").rstrip(".").strip("'").strip(":")



# -

# Go through the different questions and print the answers and the contexts

for question in questions:
    context = get_context(question)
    answer = get_answer_pipeline(question, context)
    print(f"{question[0]}\n\n{answer}\n\n{context}")
    print("-"*100)

StarcoderdataJupyterScriptsDedupFiltered