# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# +
import logging
import os
from gensim import corpora, utils
from gensim.models.wrappers.dtmmodel import DtmModel
import numpy as np

# if not os.environ.get('DTM_PATH', None):
#     raise ValueError("SKIP: You need to set the DTM path")
# -

import pandas as pd

from os import listdir
from os.path import isfile, join
mypath = '/home/avikbasu/WORK/Economics_of_Innovation/Removed_duplicates_timewise'
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]

yearwise_docs = {}
for file in onlyfiles:
    df = pd.read_csv(os.path.join(mypath,file), header=None)
    yearwise_docs[file] = list(df[10])
    

# +
#remove stopwords
import re,nltk
from nltk.tokenize import RegexpTokenizer
from nltk.stem.porter import PorterStemmer
from gensim import corpora, models
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import wordnet, stopwords
import gensim
import os
import csv
import matplotlib as plt

tag_to_type = {'J': wordnet.ADJ, 'V': wordnet.VERB, 'R': wordnet.ADV}

def get_wordnet_pos(treebank_tag):
    return tag_to_type.get(treebank_tag[:1], wordnet.NOUN)

## from <NAME>
def cleanData(doc):
    shortword = re.compile(r'\W*\b\w{1,3}\b')
    nonan = re.compile(r'[^a-zA-Z ]')
#     tokenizer = RegexpTokenizer(r'\w+')
    stop = stopwords.words('english') + ['paper','system']
    lmtzr = WordNetLemmatizer()

    tag_to_type = {'J': wordnet.ADJ, 'V': wordnet.VERB, 'R': wordnet.ADV}
    tokens = nltk.word_tokenize(shortword.sub('',nonan.sub('',doc.lower())))
    tokens = [token for token in tokens if not token in stop]
    #print tokens
    tags = nltk.pos_tag(tokens)
    is_noun = lambda pos: pos[:2] == 'NN' or pos[:2] == 'NNP' or pos[:2] == 'NNS' or pos[:2] == 'NNPS'
    tokenized = nltk.word_tokenize(lines)
    nouns = [word for (word, pos) in nltk.pos_tag(tokenized) if is_noun(pos)] 
    #print tags
    #raw_input()

    finalTokens = []
    for word, tag in zip(tokens, tags):
        finalTokens.append(lmtzr.lemmatize(word, get_wordnet_pos(tag[1])))
    return finalTokens



# -

clean_yearwise = {}
for year in yearwise_docs:
    print "Working on year {}".format(year)
    new_docs = []
    for doc in yearwise_docs[year]:
        clean_doc = cleanData(doc)
        new_docs.append(clean_doc)
    clean_yearwise[year] = new_docs


import pickle as pkl
with open('cleaned_yearwise.pkl','wb') as fp:
    pkl.dump(clean_yearwise, fp)
     

yearwise_docs['2001.csv']



StarcoderdataJupyterScriptsDedupFiltered