# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [Root]
#     language: python
#     name: Python [Root]
# ---

import glob
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import pickle
from IPython.core.debugger import set_trace
import difflib
import csv
from scipy.stats import entropy
import re

alldata_new.keys()

# +
# load data

alldata_new = pickle.load(open('output/dtm_processed_output.p', 'rb'))
doctitles=alldata_new['docnames']
doc_year=alldata_new['docs_per_year']
doc_ids =[0]+list(np.cumsum(doc_year))

term_topic = alldata_new['term_topic']# term_topic is n_years*n_topics*n_terms
terms = alldata_new['terms']
term_frequency = alldata_new['term_frequency'][1:] # weirdly the first entry is empty
doc_topicyrs = alldata_new['doc_topic']

doc_topic = []
doc_length=[]
for year in range(len(term_topic)):    
    doc_topic.append(alldata_new['doc_topic'][doc_ids[year]:doc_ids[year+1]])# doc_topic is nyear*n_docs given year*n_topics
    doc_length.append(alldata_new['doc_length'][doc_ids[year]:doc_ids[year+1]]) #doc_length is nyear*n_docs given year"""    
# rename topics by the hand-picked names
topic_labels = pickle.load(open('topicnames.p','rb'))
    
# -

def stringdiff(a,b):
    return sum ( a[i] != b[i] for i in range(len(a)) )


# +
def getlist(titles,doctitles):
    doclist=[]
    titlelist=[]
    titles = [k.lower() for k in titles]
    for doc in doctitles:
        for title in titles:
            matchratio = difflib.SequenceMatcher(None,title,doc).ratio() 
            if matchratio >.7:
                print(doc+'\n'+title)
                doclist.append(doctitles.index(doc))
                titlelist.append(title)
        if set(titlelist)==set(titles):
            break
                
    for t in titles:
        if t not in titlelist:
            print('\ncannot find: '+t)
    return([doclist,titlelist])

# given a list of paper, what are their main topics? for analyzing like a lab or an author
def maintopics(doclist,*topic_labels):
    ntopics=20
    doc_topfreq=np.empty((len(doclist),ntopics))
    for k in range(len(doclist)):
        if isinstance(doclist[0],int):
            doc_topfreq[k]=alldata_new['doc_topic'][doclist[k]]
        elif len(doclist[0])==2: # year then index
            try:
                doc_topfreq[k]=doc_topic[doclist[k][0]][doclist[k][1]]
            except:
                year=doclist[k][0]
                
                print('year%d'%doclist[k][0])
                print(len(doc_topic[year]))
                docdir = 'text_data/volume_{}/'.format(22+year)
                alldocs = glob.glob(docdir+'*.txt')
                print(len(alldocs))
                set_trace()
    doc_topfreq = np.mean(doc_topfreq,axis=0)
    doc_topfreq = doc_topfreq/sum(doc_topfreq)
    
    maintopid = np.argsort(-doc_topfreq) 
    doc_topfreq=doc_topfreq[maintopid]
    if topic_labels:
        maintopics=[topic_labels[0][idx] for idx in maintopid]
    return (maintopics,doc_topfreq)
def lab_summary(titles,doctitles,label):
    [doclist,titlelist]=getlist(titles,doctitles)        
    (mtops,meantpfreq)=maintopics(doclist,topic_labels)
    with open('result/lab_topic/'+label+'.txt','w') as f:
        for k in range(len(mtops)):
            f.write(mtops[k]+', freq={}'.format(meantpfreq[k])+'\n')
        f.write('\n papers included:\n')
        for title in titlelist:
            f.write(title)

    with open('result/lab_topic/'+label+'.csv','w') as f:
        csvwriter = csv.writer(f)
        for k in range(len(mtops)):
            csvwriter.writerow([mtops[k],meantpfreq[k]])
    return(doclist,titlelist,mtops,meantpfreq)
def labentropy(doclist):
    entrop=[]
    for doc in doclist:
        entrop.append(entropy(alldata_new['doc_topic'][doc]))
    return (np.mean(entrop))


# -

label='alex'
titles=['Computationally reproducible experiments','The Attentional Learning Trap and How to Avoid it','Online Experiments using jsPsych, psiTurk, and Amazon Mechanical Turk']
(doclist,titlelist,mtops,meantpfreq)=lab_summary(titles,label)

label='anselm'
titles=['Asking and evaluating natural language questions']
(doclist,titlelist,mtops,meantpfreq)=lab_summary(titles,label)

# +
# find the paper index for given titles
label='gureckis'
# find all titles from gureckis lab
titles = []
with open('lab_paper/Gureckis','r') as f:
    for line in iter(f.readline, ''):
        if 'Annual Conference of the Cognitive Science' in line:
            line = line.lower()
            ids = re.search(r"(20[0-1][0-9])", line).end(0)+2
            if '"' in line[ids:ids+4]:
                ids +=1
                ide = line.find('" in ')
            else:
                ide = line.find('proceedings')-2
            pptitle = line[ids:ide]
            titles.append(pptitle)
            
(doclist,titlelist,mtops,meantpfreq)=lab_summary(titles,doctitles,label)
gureckis_width=labentropy(doclist)
pickle.dump([doclist,titlelist,mtops,meantpfreq,gureckis_width],open('result/lab_topic/'+label+'.p','wb'))
# -

mtops

label='m_frank'
[doclist,titlelist,mtops,meantpfreq,gureckis_width]=pickle.load(open('result/lab_topic/'+label+'.p','rb'))
with open('result/lab_topic/'+label+'.csv','w') as f:
    csvwriter = csv.writer(f)
    for k in range(len(mtops)):
        csvwriter.writerow([mtops[k],meantpfreq[k]])


print('%d of %d papers are found for %s lab'%(len(doclist),len(titles),label))

label='m_frank'
# find all titles from <NAME>
titles = []
with open('lab_paper/M_Frank','r') as f:
    for line in iter(f.readline, ''):
        if 'Annual Conference of the Cognitive Science' in line:
            ids = line.find('). ')+3
            ide = line.find('Proceedings')-2
            pptitle = line[ids:ide]
            titles.append(pptitle)
# lab summary in the topic space
(doclist,titlelist,mtops,meantpfreq)=lab_summary(titles,doctitles,label)
frank_width=labentropy(doclist)
pickle.dump([doclist,titlelist,mtops,meantpfreq,gureckis_width],open('result/lab_topic/'+label+'.p','wb'))
print(frank_width)
print(entropy(meantpfreq))

mtops

print('%d of %d papers are found for %s lab'%(len(doclist),len(titles),label))

label='Murphy'
titles=['A knowledge resonance (KRES) model of category learning','Eyetracking as an implicit measure of category-based induction']
(doclist,titlelist,mtops,meantpfreq)=lab_summary(titles,label)
murphy_width=labentropy(doclist)

StarcoderdataJupyterScriptsDedupFiltered