# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import os

directory = 'data/multirc/'

train = pd.read_json(directory + 'train.jsonl', lines=True)
# train.info()

test = pd.read_json(directory + 'test.jsonl', lines=True)
# test.info()

val = pd.read_json(directory + 'val.jsonl', lines=True)
# val.info()

[i for i in train['evidences']]


def count_rationale_len(df):
    rationale_len = []
    for doc_evidences in df['evidences']:
        for evidence in doc_evidences[0]:
            a = evidence['end_token']-evidence['start_token']
            b = len(evidence['text'].split())
            assert(a == b)
            rationale_len.append(a)
    return rationale_len


def count_rationale_len(df):
    rationale_len = []
    for doc_evidences in df['evidences']:
        l = 0
        for evidence in doc_evidences[0]:
            a = evidence['end_token']-evidence['start_token']
            b = len(evidence['text'].split())
            assert(a == b)
            l += a
        rationale_len.append(l)
    return rationale_len


rationale_lens = count_rationale_len(train) + count_rationale_len(test) + count_rationale_len(val)
sum(rationale_lens)/len(rationale_lens)


def file_len(file):
    file = open(file, 'rb')   
    f_len = 0
    for line in file.readlines():
        f_len += len(line.rstrip().split())
    return f_len


f_lens = []
data_dir = directory + 'docs'
for filename in os.listdir(data_dir):
    f = os.path.join(data_dir, filename)
    if os.path.isfile(f):
        f_lens.append(file_len(f))
sum(f_lens)/len(f_lens)

sum(rationale_lens)/sum(f_lens)



StarcoderdataJupyterScriptsDedupFiltered