<jupyter_start><jupyter_text>CorEx on Positive Articles<jupyter_code>import pandas as pd

wri = pd.read_csv("wri.csv",index_col=0)

wri = wri[wri['labels']=='positive']
wri.reset_index(drop=True, inplace=True)

print(wri.head())
print(wri.info())
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(
    max_df=.5,
    min_df=2,
    max_features=None,
    ngram_range=(1, 2),
    norm=None,
    binary=True,
    use_idf=False,
    sublinear_tf=False,
    strip_accents = 'unicode',
    stop_words = 'english'
)

vectorizer = vectorizer.fit(wri['text'])
tfidf = vectorizer.transform(wri['text'])
vocab = vectorizer.get_feature_names()
print(len(vocab))
print(vocab)

from corextopic import corextopic as ct
TOPICS = 100
NBR_OF_WORDS = 7
anchors = []
model = ct.Corex(n_hidden=TOPICS, seed=42)
model = model.fit(
    tfidf,
    words=vocab
)<jupyter_output><empty_output><jupyter_text>['land', 'acre','hectares', 'acquisition', 'land acquisition', 'agricultural', 'acres', 'degradation','landslides','property','resettlement'],   ['farmer', 'farming', 'agricultural', 'produce', 'crop', 'crops', 'agrarian', 'farms','farm','field','fields','soil','sugarcane','vegetables','farmers','agriculture','tractor','prices crops', 'debt','quota','food','fruits','livestock','cow','wheat','harvest','harvesting','horticulture','loan','loans','milk','paddy','rice','plant','plants','potatoes','potato'],   ['mining', 'coal', 'miner', 'miners','sand mining', 'sand','bauxite','iron ore','limestone','manganese ore','granite'],   ['forest','forests', 'forest department', 'reserve', 'forest officials','forestry'],   ['animal','leopard','leopards', 'animals', 'wildlife', 'tiger', 'attacked', 'slaughter', 'lion','lions', 'threat', 'tigress', 'bear','birds','cat','cattle','crocodile','elephant','elephants','pangolin','pangolins','species'],   ['drought', 'droughts','monsoon', 'rain','rains','rainfall','disaster'],   ['water', 'irrigation', 'monsoon', 'rain', 'flood', 'floods', 'flooded', 'climate change','climate','dam','dams','drinking']<jupyter_code># Anchors designed to nudge the model towards measuring specific genres
anchors = [
    ['land','resettlement','degradation','plot'],
    ['farm','Farmers','crop','agriculture','crops','agrarian','farmer','farmers''cows','tractor','acre','fields','livestock','harvest','harvesting','potato','sugarcane','paddy','rice','milk'],
    ['mining', 'coal', 'miner', 'miners','sand mining', 'sand','bauxite','iron ore','limestone','manganese ore','granite'],
    ['forest','deforestation','trees'],
    ['animal','attacked','leopard','leopards','tiger','tigress','crocodile'],
    ['drought','rain','climate change'],
    ['water','dams','irrigation','flood','drinking']   
   
]
anchors = [
    [a for a in topic if a in vocab]
    for topic in anchors
]

model = ct.Corex(n_hidden=TOPICS, seed=42)
model = model.fit(
    tfidf,
    words=vocab,
    anchors=anchors, # Pass the anchors in here
    anchor_strength=100 # Tell the model how much it should rely on the anchors
)
for i, topic_ngrams in enumerate(model.get_topics(n_words=NBR_OF_WORDS)):
    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]
    print("Topic #{}: {}".format(i+1, ", ".join(topic_ngrams)))
topic_df = pd.DataFrame(
    model.transform(tfidf), 
    columns=["topic_{}".format(i+1) for i in range(TOPICS)]
).astype(float)
topic_df.index = wri.index
wri = pd.concat([wri, topic_df], axis=1)
for i in range(TOPICS):
    column='topic_{}'.format(i+1)
    print(wri[column].value_counts(normalize=True))
#Topic Flags
wri['topic']=wri['topic_1']+wri['topic_2']+wri['topic_3']+wri['topic_4']+wri['topic_5']+wri['topic_6']+wri['topic_7']
wri['topic'].value_counts(normalize=True)
MisTagged = wri[wri['topic']==0]
MisTagged.to_csv("MissTagged.csv")
print(MisTagged)
print(MisTagged.info())

for i in range(TOPICS):
    column='topic_{}'.format(i+1)
    print(MisTagged[column].value_counts(normalize=True))<jupyter_output>0.0    1.0
Name: topic_1, dtype: float64
0.0    1.0
Name: topic_2, dtype: float64
0.0    1.0
Name: topic_3, dtype: float64
0.0    1.0
Name: topic_4, dtype: float64
0.0    1.0
Name: topic_5, dtype: float64
0.0    1.0
Name: topic_6, dtype: float64
0.0    1.0
Name: topic_7, dtype: float64
0.0    0.822222
1.0    0.177778
Name: topic_8, dtype: float64
0.0    0.922222
1.0    0.077778
Name: topic_9, dtype: float64
0.0    0.833333
1.0    0.166667
Name: topic_10, dtype: float64
0.0    0.866667
1.0    0.133333
Name: topic_11, dtype: float64
0.0    0.844444
1.0    0.155556
Name: topic_12, dtype: float64
0.0    0.866667
1.0    0.133333
Name: topic_13, dtype: float64
0.0    0.944444
1.0    0.055556
Name: topic_14, dtype: float64
0.0    0.777778
1.0    0.222222
Name: topic_15, dtype: float64
0.0    0.777778
1.0    0.222222
Name: topic_16, dtype: float64
0.0    0.877778
1.0    0.122222
Name: topic_17, dtype: float64
0.0    0.744444
1.0    0.255556
Name: topic_18, dtype: float64
0.0    0.877778
1.0   [...]
StarcoderdataJupyterStructuredCleanDedup