<jupyter_start><jupyter_text>SVM with queue imbalance<jupyter_code>df_res = {}
for s in kernels:
    df_res_temp = pd.read_csv('../svm_queue_imbalance/res_svm/svm_linear_{}_len{}.csv'.format(
        stocks[0], data_length))
    df_res_temp = df_res_temp.append(
         pd.read_csv('../svm_queue_imbalance/res_svm/svm_sigmoid_{}_len{}.csv'.format(
             stocks[0], data_length)))
    df_res_temp = df_res_temp.append(
        pd.read_csv('../svm_queue_imbalance/res_svm/svm_rbf_{}_len{}.csv'.format(
            stocks[0], data_length)))
    df_res[s] = df_res_temp
    df_res[s].index = list(range(len(df_res[s])))

df_best_svm = pd.DataFrame()

idx_max = df_res[s].sort_values(by='matthews', ascending=False).groupby(
    'kernel')['matthews'].idxmax()
df_best_svm = df_best_svm.append(df_res[s].loc[idx_max])
df_best_svm
 from sklearn import utils

def get_classes_weights(y_train):
    classes = np.unique(y_train)
    class_weight_list = utils.class_weight.compute_class_weight('balanced', classes, y_train)
    class_weights = {classes[0]: class_weight_list[0], classes[1]: class_weight_list[1]}
    return class_weights

def fit_best_svm_classifier(df_best_svm, df, kernel=None):    
    gamma = df_best_svm[df_best_svm['kernel'] == kernel]['gamma'].values[0]
    coef0 = df_best_svm[df_best_svm['kernel'] == kernel]['coef0'].values[0]
    c = df_best_svm[df_best_svm['kernel'] == kernel]['C'].values[0]

    X = df['queue_imbalance'].values.reshape(-1, 1)
    y = df['mid_price_indicator']
   
    weights = get_classes_weights(y)
    clf = SVC(gamma=gamma, C=c, coef0=coef0, kernel=kernel, random_state=23131, class_weight=weights)
    clf.fit(X, y)
    return clf
    
def get_scores_dict_for_data(functions_to_run, dfs, log_clf, stock):
    scores = {'stock': stock}
    for func_name, func in functions_to_run.items():
        for df_name, df in dfs.items():
            pred = log_clf.predict(df['queue_imbalance'].values.reshape(-1, 1))
            df[f'pred'] = pred
            scores['{}_{}'.format(df_name, func_name)] = func(df['mid_price_indicator'], pred)
    return scores
            
functions_to_run = {'precision': metrics.precision_score, 'roc_auc': metrics.roc_auc_score,
                   'f1_score': metrics.f1_score, 'recall': metrics.recall_score, 
                   'matthews': metrics.matthews_corrcoef, 'kappa': metrics.cohen_kappa_score}
scores = []

for kernel in df_best_svm['kernel'].unique():
    stock = stocks[0]
    log_clf = fit_best_svm_classifier(df_best_svm, d_stocks[kernel], kernel=kernel)
    dfs = {'train': d_stocks[kernel], 'test': d_test_stocks[kernel], }
    res_validation = model.validate_model(
        log_clf, 
        d_stocks[kernel][['queue_imbalance']], d_stocks[kernel]['mid_price_indicator'])
    res = get_scores_dict_for_data(functions_to_run, dfs, log_clf, stock)
    res = {**res, **res_validation}
    scores.append(res)
df_scores = pd.DataFrame(scores)
def convert_scores(df, column):
    scores = []
    for i, row in df.iterrows():
        scores.append(np.mean(row[column]))
    return scores
scores_columns = ['f1', 'kappa', 'matthews', 'precision', 'recall', 'roc_auc', 'train_f1', 'train_kappa',
       'train_matthews', 'train_precision', 'train_recall', 'train_roc_auc']

for col in scores_columns:
    df_scores[col] = convert_scores(df_scores, col)
df_scores
print('Pivot values')
for i in range(len(kernels)):
    df = d_stocks[kernels[i]]
    print(np.mean([np.min(df[df['pred'] == 1]['queue_imbalance']), 
                  np.max(df[df['pred'] == 0]['queue_imbalance'])]))
f, ax = plt.subplots(3, 1, figsize=(35, 15), sharex=True)
i = 0

for i in range(len(kernels)):
    s = kernels[i]
    df = d_stocks[s]
    X = d_stocks[s][['queue_imbalance']].values
    y = d_stocks[s]['mid_price_indicator'].values.astype(np.integer)

    clf = fit_best_svm_classifier(df_best_svm, d_stocks[s], kernel=s)
    plot_decision_regions(X[0:900], y[0:900], clf=clf,ax=ax[i], colors=','.join(['orange', 'blue']))
    ax[i].set_xlabel('Queue Imbalance')
    ax[i].set_title('SVM Decision Regions for {} on training data'.format(s))
    ax[i].set_xlim(-1.01, 1.01)
plt.tight_layout()
if should_save_fig:
    print('Saving')
    plt.savefig('svm_decision_region_4549.png')<jupyter_output>Saving
StarcoderdataJupyterStructuredCleanDedup