# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: my_py
#     language: python
#     name: my_py
# ---

# +
import pandas as pd
import os
import datetime
import pickle
import re
import time
from collections import Counter
import numpy as np

import nltk
nltk.data.path
# -

from nltk.tokenize import sent_tokenize, word_tokenize
#nltk.download('stopwords')
from nltk.corpus import stopwords

import spacy
nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])

import random


# ## 1. Import and Preprocess Data

# Define a text preprocessor (lemmatizer)
def my_preprocessor(text):
    doc=nlp(text.lower())
    lemmas=[token.lemma_ for token in doc if not token.is_punct | token.is_space]
    texts_out=" ".join(lemmas)
    return texts_out


# Import expanded reg sentences
df_regSentsExpand=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/allRegSentsExpand.pkl')
print(df_regSentsExpand.info())

# Convert reg sentences to list
regSentsExpand=df_regSentsExpand['RegSentsExpand'].tolist()
print(len(regSentsExpand), regSentsExpand[0])

# Examples
print(regSentsExpand[4031])
print(my_preprocessor(regSentsExpand[4031]))

# Preprocess all expanded reg sentences
regSentsExpand_lemmatized=[my_preprocessor(sent) for sent in regSentsExpand]

# Examples
print(len(regSentsExpand_lemmatized), regSentsExpand_lemmatized[0])

print(regSentsExpand[1],regSentsExpand_lemmatized[1])

# +
# # Save preprocessed expanded reg sentences
# with open('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/regSentsExpand_lemmatized', 'wb') as fp:
#     pickle.dump(regSentsExpand_lemmatized, fp)
# -

# ## 2. Match Noun Chunks

# Import preprocessed expanded reg sentences
with open ('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/regSentsExpand_lemmatized', 'rb') as fp:
    regSentsExpand_lemmatized = pickle.load(fp)
print(len(regSentsExpand_lemmatized), regSentsExpand_lemmatized[0])

# Use all unique noun chunks in rule titles
with open ('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/ruleTitleUniqueNounChunks', 'rb') as fp:
    nounchunks = pickle.load(fp)
print(len(nounchunks),nounchunks[0:20])

# Compile a new re pattern with all noun chunks
pattern=re.compile(r"\b"+r"\b|\b".join(map(re.escape, nounchunks))+r"\b")

# +
# Match noun chunks in all expanded reg sentences
start_time = time.time()

nounchunk_match=[]
nounchunk_match_words=[]
for sent in regSentsExpand_lemmatized:
    match_words=[]
    match=0
    find=pattern.findall(sent)
    if len(find)>0:
        match_words=find
        match=len(find)
    nounchunk_match.append(match)
    nounchunk_match_words.append(match_words)
    
print("--- %s seconds ---" % (time.time() - start_time))
# -

# Examples of matched noun chunks
print(len(nounchunk_match), len(nounchunk_match_words))
print(nounchunk_match_words[-1], nounchunk_match[-1])

print(df_regSentsExpand.head())

print(df_regSentsExpand[df_regSentsExpand['NounChunksMatch']>0].info())

# Export results: matched noun chunks in expanded reg sentences
df_regSentsExpand['NounChunksMatch']=nounchunk_match
df_regSentsExpand['NounChunkMatchWords']=nounchunk_match_words
df_regSentsExpand.to_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunks3.pkl')

# ## 3. All Noun Chunk Occurences Across All Unique Articles

df_regSentsExpand=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunks3.pkl')
print(df_regSentsExpand.info())

# Remove duplicated articles
IDs_nodup=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/IDs_no_duplicates.csv')
print(IDs_nodup.info())

df_regSentsExpand['ID']=df_regSentsExpand['ID'].astype('int64')
df_regSentsExpand=IDs_nodup.merge(df_regSentsExpand,on='ID',how='left').reset_index(drop=True)
print(df_regSentsExpand.info())

# All unique macthed noun chunks and occurences from expanded reg sentences
allMatchWords=[]
for list in df_regSentsExpand['NounChunkMatchWords']:
    allMatchWords=allMatchWords+list
print(len(allMatchWords))

print(allMatchWords[100])

allMatchWordsCount=Counter(allMatchWords)
print(allMatchWordsCount)

df_MatchWords = pd.DataFrame(allMatchWordsCount.items(),columns = ['Noun Chunks','Occurences'])
print(df_MatchWords.info())

df_MatchWords=df_MatchWords.sort_values('Occurences',ascending=False).reset_index(drop=True)
print(df_MatchWords.head())

# Export noun chunk occurences
df_MatchWords.to_csv('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunkOccurences3.csv',index=False)

# ## 4. Remove General Terms after Human Auditing

# Remove general terms from matched noun chunks in expanded reg sentences
df_MatchWordsRemove=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunkOccurences3_Remove.csv')
print(df_MatchWordsRemove.info())

# Noun chunks to be removed ("general terms")
matchwords_remove=df_MatchWordsRemove['Noun Chunks'].tolist()
print(matchwords_remove[0:20])

# Reg sections with all matched noun chunks
df_regSentsExpand=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunks3.pkl')
print(df_regSentsExpand.info())

# Remove general terms from all matched noun chunks
df_regSentsExpand['NounChunkMatchWordsFiltered']=np.nan
df_regSentsExpand['NounChunkMatchFiltered']=0
for i in range(0,len(df_regSentsExpand['NounChunkMatchWords'])):
    df_regSentsExpand['NounChunkMatchWordsFiltered'][i]=[w for w in df_regSentsExpand['NounChunkMatchWords'][i] 
                                                         if w not in matchwords_remove]
    df_regSentsExpand['NounChunkMatchFiltered'][i]=len(df_regSentsExpand['NounChunkMatchWordsFiltered'][i])
print(df_regSentsExpand.info())

print(df_regSentsExpand[df_regSentsExpand['NounChunkMatchFiltered']>0].info())

df_regSentsExpand.head()

# Check some examples
for i in range(0,100):
    print(df_regSentsExpand['NounChunkMatchWords'][i])
    print(df_regSentsExpand['NounChunkMatchWordsFiltered'][i],'\n')

df_regSentsExpand.to_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunks3.pkl')

# ## 5. Filtered Noun Chunk Occurences Acorss Regulation-related Articles

df_regSentsExpand=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunks3.pkl')
print(df_regSentsExpand.info())

# Remove duplicated articles
IDs_nodup=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/IDs_no_duplicates.csv')
print(IDs_nodup.info())

df_regSentsExpand['ID']=df_regSentsExpand['ID'].astype('int64')
df_regSentsExpand=IDs_nodup.merge(df_regSentsExpand,on='ID',how='left').reset_index(drop=True)
print(df_regSentsExpand.info())

# Regulation-related articles
df_reg=df_regSentsExpand[df_regSentsExpand['NounChunkMatchFiltered']>0].reset_index(drop=True)
print(df_reg.info())

# All unique macthed noun chunks and occurences from expanded reg sentences
allMatchWords=[]
for list in df_reg['NounChunkMatchWordsFiltered']:
    allMatchWords=allMatchWords+list
print(len(allMatchWords))

print(allMatchWords[0])

allMatchWordsCount=Counter(allMatchWords)
print(allMatchWordsCount)

df_MatchWords = pd.DataFrame(allMatchWordsCount.items(),columns = ['Noun Chunks','Occurences'])
print(df_MatchWords.info())

df_MatchWords=df_MatchWords.sort_values('Occurences',ascending=False).reset_index(drop=True)
print(df_MatchWords.head())

# Export noun chunk occurences
df_MatchWords.to_csv('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_FilteredNounChunkOccurences.csv',index=False)



StarcoderdataJupyterScriptsDedupFiltered