# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# In this example, we load up two corpora - one for training a classifier, and one for testing it.
#
# We preprocess the two corpora by extracting text parses for the utterances in them. Following this, we extract politeness features from these text parses and train a politeness classifier on these politeness features from the training corpus. We then use the trained classifier to compute the average politeness score of utterances from two subreddits (r/aww and r/politics) in the testing corpus.

import convokit

from convokit import Corpus, download, TextParser, PolitenessStrategies, Classifier

train_corpus = Corpus(filename=download('wiki-politeness-annotated'))

test_corpus = Corpus(filename=download('reddit-corpus-small'))

# ## Step 1: Preprocessing

parser = TextParser()
parser.transform(train_corpus)


parser.transform(test_corpus)

# ## Step 2: Feature extraction

ps = PolitenessStrategies()
ps.transform(train_corpus)
ps.transform(test_corpus)

# ## Step 3: Analysis

clf = Classifier(obj_type='utterance', pred_feats=['politeness_strategies'], 
                 labeller=lambda utt: utt.meta['Binary']==1)

clf.fit(train_corpus)

clf.transform(test_corpus)

aww_vals = clf.summarize(test_corpus, selector=lambda utt: utt.meta['subreddit']=='aww')
politics_vals = clf.summarize(test_corpus, selector=lambda utt: utt.meta['subreddit']=='politics')

print(aww_vals['pred_score'].mean())
print(politics_vals['pred_score'].mean())



StarcoderdataJupyterScriptsDedupFiltered