# ## Random forest model

# +
# %sh
pip install pydotplus
sudo apt-get install -y graphviz



# -

dbutils.library.restartPython()

# +

# Import libraries

from sklearn import tree

from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor
from sklearn.linear_model import LinearRegression,BayesianRidge,Ridge, Lasso, ElasticNet
from sklearn.metrics import mean_squared_error,r2_score
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeRegressor
from statsmodels.api import OLS
import matplotlib.pyplot as plt
import matplotlib.ticker as plticker
import numpy as np
import pandas as pd
import pydotplus
from re import sub
import scipy.stats as sci
import seaborn as sns
 
# Set visualization prefrences 
sns.set(font_scale=1.5, style="darkgrid")
pd.set_option('display.max_columns', None)
# -

df = spark.read.parquet('/databricks/driver/spark_merged_modeling.parquet')
#df.show(10)
df= df.toPandas()

# +
non_features = ['county','state','POP_ESTIMATE_2018','total_state_pop', 
                'pop_fraction']

to_be_normalzied_features = ['cases','deaths','All_Specialties_AAMC',
                             'Total_nurse_practitioners_2019',
                             'Total_physician_assistants_2019',
                             'Total_Hospitals_2019',
                             'Total_Primary_Care_Physicians_2019',
                             'Surgery_specialists_2019',
                             'Emergency_Medicine_specialists_2019',
                             'Total_Specialist_Physicians_2019',
                             'ICU_Beds',
                             ]



in_percentage = ['Percentage_of_Active_Physicians_Who_Are_Female_2018_AAMC',
                 'Percentage_of_Active_Physicians_Who_Are_Intertiol_Medical_Graduates_IMGs_2018_AAMC',
                 'Percentage_of_Active_Physicians_Who_Are_Age_60_or_Older_2018_AAMC',
                 'Percentage_Change_in_Student_Enrollment_at_MD_and_DO_Schools_2008_2018_AAMC',
                 'Percentage_of_MD_Students_Matriculating_In_State_AY_2018_2019_AAMC',
                 'Percentage_of_Residents_in_ACGME_Programs_Who_Are_IMGs_as_of_December_31_2018_AAMC',
                 'Percent_Change_in_Residents_and_Fellows_in_ACGME_Accredited_Programs_2008_2018_AAMC',
              'Percentage_of_Physicians_Retained_in_State_from_Undergraduate_Medical_Education_UME_2018_AAMC',
                 ]
# -

X = df[df.columns[~df.columns.isin(non_features)]].drop(['cases','deaths'], axis=1) # Exclude deaths & cases from X
y = df['deaths']

# +
# Split into training and test data
X_train, X_test, y_train, y_test = train_test_split(X, df['deaths'], test_size=0.25, random_state=10)

print('Features size for train(X,y) and test(X_test):')
print('X_train', X_train.shape, 'y_train', y_test.shape, 'X_test', X_test.shape)




# +
#Modeling
#Add lock-down length as an additional feature

# File location and type
file_location = "/FileStore/tables/States_Lockdown_Lengh.csv"
file_type = "csv"

# CSV options
infer_schema = "true"
first_row_is_header = "true"
delimiter = ","

## import States_Lock-down_Lengh  file 
# The applied options are for CSV files. For other file types, these will be ignored.
States_Lockdown_Lengh = spark.read.format(file_type) \
  .option("inferSchema", infer_schema) \
  .option("header", first_row_is_header) \
  .option("sep", delimiter) \
  .load(file_location)
display(States_Lockdown_Lengh)


# -

def regression_fit(model, X_train, y_train, X_test, y_test):
    '''Fit and evaluate a regression model'''
    # Fit model
    model.fit(X_train, y_train)
    
    # Predict test y values
    y_prediction = model.predict(X_test)

    # Calculate root-mean-square error (RMSE), indicative of diffrences between predicted and obseverd values
    RMSE = np.sqrt(mean_squared_error(y_true = y_test, y_pred = y_prediction))

    # Calcualte R square, indicative of the percentage of variance of the predicted varialbe explained by the model
    R2 = r2_score(y_test,y_prediction)

    return (model, RMSE,R2)


# Choose training variable (based on gini importance)
train_vars = ['ICU_Beds',
'Adult_obesity_percentage',
'Quality_of_Life_rank',
'Excessive_drinking_percentage',
'Health_Behaviors_rank',
'Population_per_sq_mile',
'Length_of_Life_rank',
'Clinical_Care_rank',
'Adult_smoking_percentage',
'Total_Specialist_Physicians_2019',
'Physical_Environment_rank'
]

# +
# Set regresssion models for testing
reg_models = list()

reg_models = [
    ('Linear regression',LinearRegression()),
    ('Bayesian Ridge regression', BayesianRidge()),
    ('Decision tree',DecisionTreeRegressor(max_depth=1500,random_state=0)),
    ('Random forest',RandomForestRegressor(max_depth=1500,random_state=0)),
    ('Gradient boosting',GradientBoostingRegressor(random_state=0)),
    ('Linear regression with Ridge regularization', Ridge()),
    ('Linear regression with Lasso regularization (L1 penalty)',Lasso(random_state=0)),
    ('Linear regression with ElasticNet regularization',ElasticNet(random_state=0))
     ]

# +
# Train models and get test results.
results = pd.DataFrame(columns = ['Model','RMSE','R-squared'])
models = []

for i,model in enumerate(reg_models):
    trained_model, RMSE, R2 = regression_fit(model[1], X_train[train_vars], y_train, X_test[train_vars], y_test)
    results.loc[i] = [model[0],RMSE,R2]
    models.append(trained_model)
# -

results.sort_values(by='R-squared', ascending=False)

# +
mlp = MLPRegressor(hidden_layer_sizes=(5,5,5), activation='logistic', 
                   learning_rate='adaptive', solver='lbfgs', max_iter=2000, 
                   learning_rate_init=.001, random_state=10)

mlp.fit(X_train[train_vars],y_train)

predict_train = mlp.predict(X_train[train_vars])
predict_test = mlp.predict(X_test[train_vars])

# +
# Calculate root-mean-square error (RMSE), indicative of diffrences between predicted and obseverd values
RMSE = np.sqrt(mean_squared_error(y_true = y_test, y_pred = predict_test))

# Calcualte R square, indicative of the percentage of variance of the predicted varialbe explained by the model
R2 = r2_score(y_test,predict_test)

print(RMSE,R2)
results.loc[i+1] = ['Multilayer Perceptron', RMSE, R2]


# +
# Train a neural netowrk (MLP)

mlp = MLPRegressor(hidden_layer_sizes=(5,5,5), activation='logistic', 
                   learning_rate='adaptive', solver='lbfgs', max_iter=2000, 
                   learning_rate_init=.001, random_state=10)

mlp.fit(X_train[train_vars],y_train)

predict_train = mlp.predict(X_train[train_vars])
predict_test = mlp.predict(X_test[train_vars])

# +
# Calculate root-mean-square error (RMSE), indicative of diffrences between predicted and obseverd values
RMSE = np.sqrt(mean_squared_error(y_true = y_test, y_pred = predict_test))

# Calcualte R square, indicative of the percentage of variance of the predicted varialbe explained by the model
R2 = r2_score(y_test,predict_test)

print(RMSE,R2)
results.loc[i+1] = ['Multilayer Perceptron', RMSE, R2]

# +
# Plot an example tree from the Gradient boosting model
from IPython.display import Image
# %matplotlib inline
from IPython.core.display import HTML 
from IPython.display import display
#from pyspark.ml.image import ImageSchema
sub_tree =  models[4].estimators_[10, 0]
dot_data = tree.export_graphviz(
    sub_tree,
    out_file=None, filled=True,
    rounded=True,  
    special_characters=True,
    proportion=True,
    feature_names=X_train[train_vars].columns
)

graph = pydotplus.graph_from_dot_data(dot_data)  
#Image(graph.create_png())
Image(graph.create_png()).show()
# -

im = Image(graph.create_png()).visualize()
open('output.png', 'wb').write(im.data)



StarcoderdataJupyterScriptsDedupFiltered