<jupyter_start><jupyter_text>Load data<jupyter_code>classifier = cPickle.load(open("model/classifier.pickle"))
rawtext = cPickle.load(open("model/textdata/rawtext.pickle"))
BoW = cPickle.load(open("model/BoW.pickle"))
BoW_transformer = cPickle.load(open("model/BoW_transformer.pickle"))
labeled_texts = [[party, text] for (party, texts) in rawtext.items() for text in texts]
df = pd.DataFrame(labeled_texts, columns=["party", "text"])
df
import sklearn
# find out how sklearn arranges feature vectors
c = sklearn.feature_extraction.text.CountVectorizer()
c.fit(['foo', 'bar']).transform(['foo', 'foo bar bar']).toarray()
c.get_feature_names()
df['tfidf'] = [v for (party, matrix) in BoW.items() for v in matrix.toarray()]
df<jupyter_output><empty_output><jupyter_text>Sample (for quicker computations)<jupyter_code>rows = np.random.choice(df.index.values, 200)
df_sample = df.ix[rows].copy()
df_sample
df_sample.groupby('party').count()['text'].plot(kind='bar')<jupyter_output><empty_output><jupyter_text>MDS<jupyter_code>clf = manifold.MDS(n_components=2, n_init=1, max_iter=100)
X_mds = clf.fit_transform(df_sample['tfidf'].tolist())
x, y = np.transpose(X_mds)
df_sample['mds_x'] = x
df_sample['mds_y'] = y
fig = None
colors = dict(spd='red', cdu='black', gruene='green', linke='orange')
for party, group in df_sample.groupby('party'):
    _fig = group.plot(kind='scatter', x='mds_x', y='mds_y', label=party, color=colors[party], ax=fig)
    fig = _fig<jupyter_output><empty_output><jupyter_text>t-SNE<jupyter_code>tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
X_tsne = clf.fit_transform(df_sample['tfidf'].tolist())
x, y = np.transpose(X_tsne)
df_sample['tsne_x'] = x
df_sample['tsne_y'] = y
fig = None
colors = dict(spd='red', cdu='black', gruene='green', linke='orange')
for party, group in df_sample.groupby('party'):
    _fig = group.plot(kind='scatter', x='tsne_x', y='tsne_y', label=party, color=colors[party], ax=fig)
    fig = _fig<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>X_pca = sklearn.decomposition.TruncatedSVD(n_components=2).fit_transform(df_sample['tfidf'].tolist())
x, y = np.transpose(X_pca)
df_sample['pca_x'] = x
df_sample['pca_y'] = y
fig = None
colors = dict(spd='red', cdu='black', gruene='green', linke='orange')
for party, group in df_sample.groupby('party'):
    _fig = group.plot(kind='scatter', x='pca_x', y='pca_y', label=party, color=colors[party], ax=fig)
    fig = _fig<jupyter_output><empty_output><jupyter_text>With predictions...<jupyter_code>clf = classifier['classifier']
labels = classifier['labels']
df_sample['probas'] = map(list, clf.predict_proba(df_sample['tfidf'].tolist()))
def predicted_label(probas):
    return labels[probas.index(max(probas))]
    
df_sample['predicted_party'] = df_sample['probas'].map(predicted_label)
clf = manifold.MDS(n_components=2, n_init=1, max_iter=100)
X_mds = clf.fit_transform(df_sample['probas'].tolist())
x, y = np.transpose(X_mds)
df_sample['probas_mds_x'] = x
df_sample['probas_mds_y'] = y
df_sample
fig = None
colors = dict(spd='red', cdu='black', gruene='green', linke='orange') # change to predicted class?
for party, group in df_sample.groupby('party'):
    _fig = group.plot(kind='scatter', x='probas_mds_x', y='probas_mds_y', label=party, color=colors[party], ax=fig)
    fig = _fig
fig = None
colors = dict(spd='red', cdu='black', gruene='green', linke='orange') # change to predicted class?
for party, group in df_sample.groupby('predicted_party'):
    _fig = group.plot(kind='scatter', x='probas_mds_x', y='probas_mds_y', label=party, color=colors[party], ax=fig)
    fig = _fig<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup