<jupyter_start><jupyter_text>DREAM5 - SGBM, denormalized variable importance<jupyter_code>import os
import sys
sys.path.append('../../')

from arboreto.core import *
from arboreto.utils import *

import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Data paths<jupyter_code>wd = os.getcwd().split('arboreto')[0] + 'arboreto/resources/dream5/'

net1_ex_path = wd + 'net1/net1_expression_data.tsv'
net1_tf_path = wd + 'net1/net1_transcription_factors.tsv'

net3_ex_path = wd + 'net3/net3_expression_data.tsv'
net3_tf_path = wd + 'net3/net3_transcription_factors.tsv'

net4_ex_path = wd + 'net4/net4_expression_data.tsv'
net4_tf_path = wd + 'net4/net4_transcription_factors.tsv'<jupyter_output><empty_output><jupyter_text>Dask client<jupyter_code>from dask.distributed import Client, LocalCluster
client = Client(LocalCluster())
client<jupyter_output><empty_output><jupyter_text>Use port forwarding to view the dashboard `$ ssh -L 8000:localhost:8787 nostromo`<jupyter_code>client.shutdown()<jupyter_output><empty_output><jupyter_text>Infer networks (with meta)<jupyter_code>def infer_network_and_meta(ex_path, tf_path, client):
    ex_matrix = load_expression_matrix(ex_path)
    gene_names = load_gene_names(ex_path)
    tf_names = load_tf_names(tf_path, gene_names)
    
    network_graph, meta_graph = create_graph(ex_matrix,
                                             gene_names,
                                             tf_names,
                                             "GBM",
                                             SGBM_KWARGS,
                                             'all',  # [200, 201, 202, 203, 204],
                                             early_stop_window_length=25,
                                             include_meta=True)
    
    result = client.compute([network_graph, meta_graph], sync=True)
    
    network_df = result[0]
    meta_df = result[1]
    
    return network_df, meta_df
%%time
net1, meta1 = infer_network_and_meta(net1_ex_path, net1_tf_path, client)
len(net1)
meta1.head()
meta1.hist(bins=50)
plt.show()<jupyter_output><empty_output><jupyter_text>Correlate `n_estimators` with max `importance`<jupyter_code># net1['max_importance'] = net1.groupby(['target'])['importance'].transform(max)
# max_by_target_df = net1.groupby(['target'])['importance'].max().reset_index()
# top_3_by_target_df = net1.groupby(['target'])['importance'].nlargest(3).reset_index()
# net1.groupby(['target'])['importance'].nlargest(5).reset_index()
# net1_max_importance_by_n_rounds = meta1.merge(net1[['target', 'max_importance']].drop_duplicates(), how='left', on=['target'])
# net1_max_importance_by_n_rounds = meta1.merge(max_by_target_df, how='left', on=['target'])
n = 1
meta1.merge(net1.groupby(['target'])['importance'].nlargest(n).reset_index(), 
            how='left', 
            on=['target']).plot.scatter(x='n_estimators', 
                                        y='importance', 
                                        figsize=(16, 9))
plt.show()
# cmap values: jet, coolwarm
# https://matplotlib.org/examples/color/colormaps_reference.html

n = 10
meta1.merge(net1.groupby(['target'])['importance'].nlargest(n).reset_index(), 
            how='left', 
            on=['target']).plot.hexbin(x='n_estimators', 
                                       y='importance',   
                                       bins='log',
                                       cmap=plt.get_cmap("inferno"),
                                       figsize=(16, 9))
plt.show()<jupyter_output><empty_output><jupyter_text>DAFUQ?* what the hell does this mean? --> probably that the normalization doesn't make a lot of sense<jupyter_code>net1_max_importance_by_n_rounds.plot.scatter(x='n_estimators', y='max_importance', figsize=(10, 7))
plt.show()
net1_max_importance_by_n_rounds.plot.scatter(x='value', y='max_importance', figsize=(10, 7))
plt.show()
meta1.hist(bins=50, figsize=(10,7))
plt.show()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup