# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernel_info:
#     name: python3-azureml
#   kernelspec:
#     display_name: PyCharm (drug-resistance-prediction-cambiohack)
#     language: python
#     name: pycharm-f27e5417
# ---

# + gather={"logged": 1604307425503}
import pandas as pd

import warnings
warnings.filterwarnings('ignore')


import h2o
h2o.init(min_mem_size='27G')



# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604307447638}
binarized_final_df = h2o.import_file("../data/processed/final.binarized_final_monolabel_df.tsv")
binarized_final_df.head()

# + gather={"logged": 1604307464844}

train = h2o.import_file("../data/processed/final.train.tsv")
train.head()

# + gather={"logged": 1604307478044}

test = h2o.import_file("../data/processed/final.test.tsv")
test.head()

# + gather={"logged": 1604307478173}
# Identify predictors and response
train_predictor_cols = train.columns
train_response_col = "Resistance_Status"
train_predictor_cols.remove('SampleID')
train_predictor_cols.remove(train_response_col)
print("train frame - predictor column: ", train_predictor_cols[0], train_predictor_cols[-1])
print("train frame - response column: ", train_response_col)



# Identify predictors and response
test_predictor_cols = test.columns
test_response_col = "Resistance_Status"
test_predictor_cols.remove('SampleID')
test_predictor_cols.remove(test_response_col)
print("test frame - predictor columns: ", test_predictor_cols[0], test_predictor_cols[-1])
print("test frame - response column: ", test_response_col)

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604307478223}
# For binary classification, response should be a factor
train[train_response_col] = train[train_response_col].asfactor()
test[test_response_col] = test[test_response_col].asfactor()

x = train_predictor_cols
y = train_response_col


# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604307479662}
test.head()

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604307481326}
train.head()

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604302962344}
from h2o.estimators import H2OPrincipalComponentAnalysisEstimator

my_pca = H2OPrincipalComponentAnalysisEstimator(
                                                   k = 400,
                                                #    pca_method = "gram_s_v_d",

                                                   # use_all_factor_levels = True,
                                                   # pca_method = "glrm",
)


# TODO: Do PCA for the entire dataset, it doesn't make sense for test/train only
my_pca.train(x=x, y=y, training_frame=binarized_final_df)

# Try out with smaller test dataset first
# my_pca.train(x=x, y=y, training_frame=test)

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604300649929}
# save the model
model_path = h2o.save_model(model= my_pca, path="../models/my_pca_model", force=True)

# print(model_path)
# model_path = "../data/processed/my_rf_model/DRF_model_python_1601519217841_129"
    
# load the model
# my_rf = h2o.load_model(model_path)

# download the model built above to your local machine
# my_local_model = h2o.download_model(my_rf, path="../data/processed/my_rf_model")

# upload the model that you just downloded above
# to the H2O cluster
# uploaded_model = h2o.upload_model(my_local_model)

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1603964997073}
# Generate predictions on a test set (if neccessary)
pred = my_pca.predict(binarized_final_df)

pred

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604300650178}
my_pca.summary()

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604300650477}
my_pca.show()

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604300657396}
my_pca.model_performance(test)

# + [markdown] nteract={"transient": {"deleting": false}}
# # Save the PCA dataframe

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604300671783}
# Generate predictions
train_pca = my_pca.predict(train)
h2o.export_file(frame=train_pca, path="../data/processed/final.train.pca400.tsv", force=True)
train_pca.head()



# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604300695079}
# Generate predictions
test_pca = my_pca.predict(test)
h2o.export_file(frame=test_pca, path="../data/processed/final.test.pca400.tsv", force=True)
test_pca.head()

# + [markdown] nteract={"transient": {"deleting": false}}
# # GLRM Model

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}}
from h2o.estimators import H2OGeneralizedLowRankEstimator

my_glrm = H2OGeneralizedLowRankEstimator( # k=5,
                                            # loss="quadratic",
                                            # gamma_x=0.5,
                                            # gamma_y=0.5,
                                            # max_iterations=700,
                                            # recover_svd=True,
                                            # init="SVD",
                                            # transform="standardize"
                                            )
my_glrm.train(training_frame=test)

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1604250529998}
h2o.shutdown()

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}}


StarcoderdataJupyterScriptsDedupFiltered