<jupyter_start><jupyter_text>Reaction features<jupyter_code>features_reactions = pd.DataFrame(index=cc.index)
features_reactions['n_up'] = cc['Actions.Agree.Total']
features_reactions['n_down'] = cc['Actions.Disagree.Total']
features_reactions['n_reply'] = cc['Actions.Comment.Total']
sns.pairplot(features_reactions)<jupyter_output><empty_output><jupyter_text>Post date features<jupyter_code>features_date = pd.DataFrame(index=cc.index)
features_date['t_week'] = cc.Timestamp.dt.week
features_date['t_dow'] = cc.Timestamp.dt.dayofweek
features_date['t_hour'] = cc.Timestamp.dt.hour
features_date['t_day'] = cc.Timestamp.dt.day
sns.pairplot(features_date)<jupyter_output><empty_output><jupyter_text>Spacy NLP ...<jupyter_code>import spacy                           # See "Installing spaCy"
nlp = spacy.load('en')                 # You are here.

spacy_docs = pd.DataFrame(index=cc.index)
docs = cc.Body.apply(nlp)
vec = docs.apply(lambda x: x.vector)
feature_word_vec = pd.DataFrame(vec.tolist(), columns=['spacy_%s'%i for i in range(300)])
feature_word_vec['spacy_sent'] = docs.apply(lambda x: x.sentiment)
# tfidf 
'''
Author: <NAME>
Python: 3.6.0
Date: 24/6/2017
'''
import pandas as pd
import numpy as np
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score
from scipy.sparse import csr_matrix
text = cc['Body']
# create a stopset (words that occur to many times)
stopset = set(stopwords.words('english'))
vectorizer = TfidfVectorizer(use_idf=True, lowercase=True, strip_accents='ascii', stop_words=stopset)
features_tfidf = pd.DataFrame(vectorizer.fit_transform(text).toarray())<jupyter_output><empty_output><jupyter_text>Target<jupyter_code>targets = pd.read_csv('./btc-ind.csv')
targets['date'] = pd.to_datetime(targets['Date'])
targets = targets.set_index('date')
del targets['Date']
targets.tail()
join_by_date = pd.DataFrame(index=cc.index)
join_by_date['date'] = cc.Timestamp.dt.round(freq="d")
Y_all = join_by_date.join(targets, on='date').dropna()
groups = Y_all['date']
del Y_all['date']
cols = Y_all.columns
index = Y_all.index
#Y_all = pd.DataFrame(normalize(Y_all, axis=1, norm='l2'), columns=cols, index=index)
Y_all = Y_all - Y_all.mean()
Y_all = Y_all/Y_all.std()
#Y_all.plot()<jupyter_output><empty_output><jupyter_text>Combine features<jupyter_code>#features = pd.concat([features_date, features_tfidf, features_reactions, feature_word_vec], axis=1)
features = pd.concat([features_date, features_reactions, feature_word_vec], axis=1)
X_all = features.ix[Y_all.index]
X_all.shape
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import normalize
from xgboost.sklearn import XGBRegressor
from sklearn.linear_model import LinearRegression, Lasso
rf = RandomForestRegressor(n_estimators=10, max_depth=3, criterion='mse')
xgb = XGBRegressor(n_estimators=10)
regressors = [rf, Lasso()]
target_scores = {}
for indicator in targets.columns:
    Y =Y_all[indicator]    
    for reg in regressors:
        tag = indicator+':'+str(reg)[:15]
        scores = cross_val_score(reg, X_all, Y, cv=4, groups=groups, scoring='neg_mean_squared_error')
        print np.mean(scores), tag
        target_scores[tag] = scores

cv_score = pd.DataFrame(target_scores)
ms = cv_score.mean(axis=0)
ms.sort_values(ascending=False)
indicator = 'BTC_cbrt_dv_T1:Lasso(alpha=1.0'
indicator = indicator.split(":")[0]
Y = Y_all[indicator]
reg = XGBRegressor(n_estimators=100)
reg.fit(X_all, Y)
Y_t = reg.predict(X_all)
error = abs(Y - Y_t)
error.hist()
# DROP THE BULL$HIT
itruth = error < error.quantile(0.3)
X = X_all[itruth]
Y = Y_all[indicator][itruth]
G = groups[itruth]
reg = XGBRegressor(n_estimators=100, max_depth=8)
scores = cross_val_score(reg, X, Y, cv=4, groups=G, scoring='neg_mean_squared_error')
print sorted(scores)
ax = groups.hist(figsize=(12,5))
G.hist(ax=ax)
reg = XGBRegressor(n_estimators=100, max_depth=8)
reg.fit(X,Y)
Y_ = reg.predict(X)
truth_df = pd.DataFrame({'date': G, 'Y': Y_})
def get_stats(group):
    return {'min': group.min(), 'max': group.max(), 'count': group.count(), 'mean': group.mean()}


ax = targets.BTC_cbrt_dv_T1.plot()
truth.plot(ax=ax)
truth
def drop_bs(indicator, q=0.3):
    Y = Y_all[indicator]
    reg = XGBRegressor(n_estimators=100)
    reg.fit(X_all, Y)
    Y_t = reg.predict(X_all)
    error = abs(Y - Y_t)
    error.hist()
    itruth = error < error.quantile(q)
    X = X_all[itruth]
    Y = Y_all[indicator][itruth]
    G = groups[itruth]
    reg = XGBRegressor(n_estimators=30, max_depth=5)
    scores = cross_val_score(reg, X, Y, cv=4, groups=G, scoring='neg_mean_squared_error')
    print sorted(scores)
    print "MEAN CV SCORE: ", np.mean(scores)
    
    reg = XGBRegressor(n_estimators=100, max_depth=8)
    reg.fit(X,Y)
    Y_ = reg.predict(X)
    agg = pd.Series(Y_).groupby(G)
    truthscore = agg.mean()
    impact_count = agg.count()
    truth_max = agg.max()
    return pd.DataFrame(dict(truthscore=truthscore, impact_count=impact_count, truth_max=truth_max, date=truthscore.index))
dv = drop_bs('BTC_cbrt_dv_T1', 0.4)
import json
def to_json(df, path):
    a = []
    for i,d in list(df.iterrows()):
        d = d.to_dict()
        d['date'] = str(d['date'])
        a.append(d) 
    with open(path, 'w') as f:
        json.dump(a, f)
        
to_json(dv, '../bitcoin-daily-bars/out-truth-volume.json')
impactfull = cc.ix[itruth.index][itruth]
impactfull.head()
f = 'Cryptopian.Name'
a = impactfull.groupby(f).size()
b = cc.groupby(f).size()
c = pd.DataFrame(dict(a=a,b=b))
c = c[c.a>1]
c['impact'] = c.a/c.b
c.sort_values('impact', ascending=False)
dv.truthscore.plot()
target_sc<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup