# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/"} id="nDITFUNxC8HZ" outputId="2d552fe0-5bf1-4253-9bfc-5c238ec08e38"
# !pip install -U transformers==3.0.0
# !python -m nltk.downloader punkt
# !git clone https://github.com/patil-suraj/question_generation.git

# + colab={"base_uri": "https://localhost:8080/"} id="iaBuKJ83DnQK" outputId="bed4da4e-8d3f-4206-e874-90af0a6e0bba"
# %cd question_generation

# + id="zp_fz3IrDC2T"
from pipelines import pipeline
import json


# + id="zdx5EPyDFsac"
def getPassages(filename):
    with open(filename) as file:
        data = json.load(file)
    passages = list()
    for i in data["vertices"]["paragraphs"]:
      passages.append(i["text"])
    return passages


# + id="KLeMECAHSEnr"
def write_json(data, filename):
    with open(filename, 'w') as file:
        json.dump(data, file)


# + id="rSdfuNCnQ-Wd"
def getTopics(filename, paras):
    with open(filename) as file:
        data = json.load(file)
    topics = ['' for i in range(len(data["vertices"]["topics"]))]
    edges = data["edges"]["main"]
    vertices = data["vertices"]["topics"]
    count = 0
    # print(edges)
    for e in edges:
        if e[1] == "about_concept":
          # print(e)
            topic_id = int(str(e[2])[2:])
            para_id = int(str(e[0])[2:])
            paras[para_id]=  vertices[topic_id]["text"] + "; " + paras[para_id]
            count += 1
    return paras


# + id="VAphAeBrP2CS"
def getTokenizedAnswers(context, answers):
  start_word = answers[:2]
  # print("context:",context,"answer:", answers)
  start_answer = context.index(answers)
  end_answer = start_answer + len(answers)
  return {
    "answer": answers,
    "start_answer": start_answer,
    "end_answer": end_answer
  }


# + id="g7FJ06jmF6XE" colab={"base_uri": "https://localhost:8080/"} outputId="90e63bcc-1e74-4643-fe34-974b53fe2228"
passages = getPassages("../handbook_graph.json")
passages = getTopics("../handbook_graph.json", passages)
print(passages)

# + id="39ClQXykB4sZ"
with open("../preprocessed_handbook.json") as file:
  data = json.load(file)
for id in data["paras"]:
  para = data["paras"][id]
  topic = data["topics"][str(data["edges"][id]["topic"])]
  passages.append(topic+"; "+para)

# + id="8aoDm9uXGL4N" colab={"base_uri": "https://localhost:8080/", "height": 602, "referenced_widgets": ["8f5d0ce5fdd3462ab2732570db907669", "bfaa43700f8f4328ba0710ebf53b8523", "51d8e3d7344a42d78fa45e2ad5af87b1", "c5e8b1f5eeb34cd49df55849d090066c", "602f01218f3c48fa9200d6c0fe7aaca4", "9938be99aeb04be88246a1ccf912ddbb", "3ee52a0a0233496a85f8ff96ae934978", "df3f2ec973c2455290be55f8da19dc54", "<KEY>", "<KEY>", "31d3506366a849c28666a3a875351155", "b2324b74ae644742ac14a2f1b1ecc960", "eff0fcddab48437c870305cc5deeef23", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "9b376cae289d46239a4e719af0d52e52", "83a2a67a350746dfab6ef00fb109a144", "<KEY>", "04bc6a017f7c4a26b508d8b5a2541b84", "<KEY>", "<KEY>", "<KEY>", "9d8d7daabe4448fe851e4b8c325c482b", "<KEY>", "b7a70d6d80e640eb96045513b014cd1a", "a3e9b4ee12914e80845a1fcefeb5db02", "01defc217ef6493ba4d99fb26ffd2f27", "<KEY>", "<KEY>", "<KEY>", "3d9a39cca80a4f988ddba348e77febc9", "<KEY>", "eb88df95165c4e2e96c93c3d707e39a3", "<KEY>", "3712ee3909324f26bde06a5df40ad3e4", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "9072062aeafc4ead9146240866ad76a8", "07bef828bfa545879bd9033e87be4055", "adf7d2161f5641ef99152b339aa5cdba", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "bba4f5d9f878428899471593461eabe4", "<KEY>", "<KEY>", "<KEY>", "d997f71551004912a87b328079dc2a59", "e947aa3521134d219ac15d9dee5573e4", "<KEY>", "<KEY>", "5f684b73a84d4e5d892ee9985741485d", "<KEY>", "9de543e89ccd4c88891d808e1e0abea9", "<KEY>", "1b7503762bf2468e9f982c3e121400ba", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "a31c87acb9b34c218ea94798f26c84ca", "<KEY>", "f2f35dbcd4374da49a642b51ace0a054", "<KEY>", "bc7f0a42d55a45d28b8be90e0e8aafe6", "<KEY>", "9faea5a48d464db58c2a7f0ca8a311be", "<KEY>", "8fc4d09d2248497c838553353e72dcc6", "6ca51b67eba042588aaf016682e55964", "<KEY>", "<KEY>", "<KEY>", "bf934adb5e964cef9f1deadb0bd4fb30", "e1be619e24c04ce2ad578fcf58db64c5", "4d830a30fa0146b78e57698268693eb5", "83f0ead927a647b9a5c6651bc9916e81", "debc133e391a4cba96ed9134f08492a1", "7d8155272d7049f49be1353bdaad8b82", "35f4fca8d66d4802b5612d2d762e641b", "<KEY>", "6da202313e4444d1bdd9739fa068e75c", "1e54df2be5f1444ba760e1a5f8256d6b", "d3c51999e4fc4b0fa35371ace04a217d", "<KEY>", "db689dd1d7e24eb6a581183b50df6a34"]} outputId="30cfd88d-f7bc-4537-9540-8372dbadeb73"
qg = pipeline("question-generation", model="valhalla/t5-base-qg-hl")

# + colab={"base_uri": "https://localhost:8080/"} id="SwyiQry-Gm3-" outputId="ed2e91fa-9f5e-4b10-f23c-34add4cb7fe0"
qaList = list()
count = 0
questions_dictionary = dict()
id = 0
countQuestions = 0
for p in passages:
  try:
    qaList = qg(p)
    questions_dictionary[id] = {
        'qas':dict(),
        'context':p
    }
    qid = 0
    qa = questions_dictionary[id]["qas"]
    for questions in qaList:
      countQuestions += 1
      qa[qid] = dict()
      answers = getTokenizedAnswers(p, questions["answer"])
      qa[qid]["question"] = questions["question"]
      qa[qid]["answer"] = answers
      qid += 1
    id += 1
  except:
    count += 1
    print(p)
    print('error')
    continue
print(countQuestions)

# + id="bs4MNLoRQyCB"
write_json(questions_dictionary, "training_set_nueral.json")

StarcoderdataJupyterScriptsDedupFiltered