<jupyter_start><jupyter_text>Pattern I<jupyter_code># read the excel of the data for Pattern I
data1 = os.path.join(DATA_PATH, "pattern1_data.xlsx")
df1 = pd.read_excel(data1, engine="openpyxl")
df1_edit = df1
df1_edit<jupyter_output><empty_output><jupyter_text>1.1 Polynomial Regression<jupyter_code>from sklearn.decomposition import PCA

# get feature data set and label data set
X_all_11 = df1_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_11 = df1_edit["Fe3+/Fetot(Correction)"].copy()

# do dimension reduction on the training feature data set 
pca_11 = PCA(n_components=7)
# the original dimensions 9 will be transformed into 7
X_all_reduced_11 = pca_11.fit_transform(X_all_11)
X_all_reduced_11.shape
pd.DataFrame(X_all_reduced_11)
from sklearn.preprocessing import PolynomialFeatures

# based on the reduction data, do dimension augmentation with the degree of the polynomial features 3
poly_features_11 = PolynomialFeatures(degree=3, include_bias=False)
X_all_augmented_11 = poly_features_11.fit_transform(X_all_reduced_11)
X_all_augmented_11.shape
# the columns name of the augmented data set
columns_name_11 = poly_features_11.get_feature_names()

# the augmentd data set, x0 refers to the first columns in the reduced data set
X_all_augmented_11 = pd.DataFrame(X_all_augmented_11)
X_all_augmented_11.columns = columns_name_11
X_all_augmented_11
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_11 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=13)
# divide augmented X and y into training and testing data
for train_index, test_index in split_11.split(df1_edit, df1_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_11 = X_all_augmented_11.loc[train_index]
    X_test_11 = X_all_augmented_11.loc[test_index] 
    y_train_11 = y_all_11.loc[train_index]
    y_test_11 = y_all_11.loc[test_index]
from sklearn.linear_model import ElasticNet

# Based on regularized linear regression, bulid the polynomial regression with the augmented X 
polyreg_11 = ElasticNet(alpha=0.0001, l1_ratio=0.0001, max_iter=100000, tol=0.0005)
# train the model and get the corresponding score
y_train_predict_11, y_test_predict_11, train_rmse_11, test_rmse_11, train_r2_11, test_r2_11 = \
score(polyreg_11, 'polynomial regression', 'poly',X_train_11, X_test_11, y_train_11, y_test_11)
# plot binary diagram
binary_plot(y_train = y_train_11,
            y_train_label = y_train_predict_11,
            y_test = y_test_11,
            y_test_label = y_test_predict_11,
            train_rmse = train_rmse_11,
            test_rmse = test_rmse_11,
            train_r2 = train_r2_11,
            test_r2 = test_r2_11)
save_fig("poly_reg_pattern1")<jupyter_output>----------polynomial regression----------
The RMSE on training set is 0.074
The RMSE on test set is 0.085
R2 score on training set is 0.891
R2 score on test set is 0.856
Successfully store the trained model in poly.pkl
Saving figure poly_reg_pattern1<jupyter_text>1.2 Artificial Neural Network<jupyter_code># get feature data set and label data set
X_all_12=df1_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_12 =df1_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_12 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=13)
# divide augmented X and y into training and testing data
for train_index, test_index in split_12.split(df1_edit, df1_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_12 = X_all_12.loc[train_index]
    X_test_12 = X_all_12.loc[test_index] 
    y_train_12 = y_all_12.loc[train_index]
    y_test_12 = y_all_12.loc[test_index]
from sklearn.neural_network import MLPRegressor

# build up Artificial Neural Network model
mlp_12 = MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', early_stopping=False,
                   hidden_layer_sizes=(20, 120, 20), max_iter=2000, solver='adam', tol=1e-05,
                   random_state=9)

# train the model and get the corresponding score
y_train_predict_12, y_test_predict_12, train_rmse_12, test_rmse_12, train_r2_12, test_r2_12 = \
score(mlp_12, 'Artificial Neural Network', 'mlp', X_train_12, X_test_12, y_train_12, y_test_12)

# plot binary diagram
binary_plot(y_train = y_train_12,
            y_train_label = y_train_predict_12,
            y_test = y_test_12,
            y_test_label = y_test_predict_12,
            train_rmse = train_rmse_12,
            test_rmse = test_rmse_12,
            train_r2 = train_r2_12,
            test_r2 = test_r2_12)
save_fig("ann_pattern1")<jupyter_output>----------Artificial Neural Network----------
The RMSE on training set is 0.074
The RMSE on test set is 0.081
R2 score on training set is 0.889
R2 score on test set is 0.871
Successfully store the trained model in mlp.pkl
Saving figure ann_pattern1<jupyter_text>1.3 Artificial Neural Network Ensemble<jupyter_code># get feature data set and label data set
X_all_13=df1_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_13 =df1_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_13 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=13)
# divide augmented X and y into training and testing data
for train_index, test_index in split_13.split(df1_edit, df1_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_13 = X_all_13.loc[train_index]
    X_test_13 = X_all_13.loc[test_index] 
    y_train_13 = y_all_13.loc[train_index]
    y_test_13 = y_all_13.loc[test_index]
from sklearn.ensemble import BaggingRegressor

# single ANN model with the same hyperparameter as 1.2 ANN
mlp_13 = MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', early_stopping=False,
                   hidden_layer_sizes=(20, 120, 20), max_iter=2000, solver='adam', tol=1e-05,
                   random_state=9)

# Use Bagging method to ensemble 500 ANN models
mlp_bag_13 = BaggingRegressor(mlp_13, n_estimators=500, bootstrap=True, bootstrap_features=True,
                              n_jobs=-1, oob_score=True, random_state=42)
# train the model and get the corresponding score
y_train_predict_13, y_test_predict_13, train_rmse_13, test_rmse_13, train_r2_13, test_r2_13 = \
score(mlp_bag_13, 'Artificial Neural Network Ensemble', 'mlp_bag', X_train_13, X_test_13, y_train_13, y_test_13)

# plot binary diagram
binary_plot(y_train = y_train_13,
            y_train_label = y_train_predict_13,
            y_test = y_test_13,
            y_test_label = y_test_predict_13,
            train_rmse = train_rmse_13,
            test_rmse = test_rmse_13,
            train_r2 = train_r2_13,
            test_r2 = test_r2_13)
save_fig("ann_ensemble_pattern1")<jupyter_output>----------Artificial Neural Network Ensemble----------
The RMSE on training set is 0.077
The RMSE on test set is 0.086
R2 score on training set is 0.881
R2 score on test set is 0.853
Successfully store the trained model in mlp_bag.pkl
Saving figure ann_ensemble_pattern1<jupyter_text>1.4 Decision Tree<jupyter_code># get feature data set and label data set, append a new feature column to X_all_14
X_all_14=df1_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
X_all_14['Na/Mg'] = X_all_14['Na'] / X_all_14['Mg']
y_all_14 =df1_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_14 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=13)
# divide augmented X and y into training and testing data
for train_index, test_index in split_14.split(df1_edit, df1_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_14 = X_all_14.loc[train_index]
    X_test_14 = X_all_14.loc[test_index] 
    y_train_14 = y_all_14.loc[train_index]
    y_test_14 = y_all_14.loc[test_index]
from sklearn.tree import DecisionTreeRegressor

# build up Decision Tree model
dt_14 = DecisionTreeRegressor(ccp_alpha=0.0, criterion='mse', max_depth=None,
                              max_features=None, max_leaf_nodes=None, min_impurity_decrease=0.0,
                              min_impurity_split=None, min_samples_leaf=12, min_samples_split=2,
                              min_weight_fraction_leaf=0.0, presort='deprecated', random_state=42,
                              splitter='best')

# train the model and get the corresponding score
y_train_predict_14, y_test_predict_14, train_rmse_14, test_rmse_14, train_r2_14, test_r2_14 = \
score(dt_14, 'Decision Tree', 'dt', X_train_14, X_test_14, y_train_14, y_test_14)

# plot binary diagram
binary_plot(y_train = y_train_14,
            y_train_label = y_train_predict_14,
            y_test = y_test_14,
            y_test_label = y_test_predict_14,
            train_rmse = train_rmse_14,
            test_rmse = test_rmse_14,
            train_r2 = train_r2_14,
            test_r2 = test_r2_14)
save_fig("dt_pattern1")<jupyter_output>----------Decision Tree----------
The RMSE on training set is 0.07
The RMSE on test set is 0.08
R2 score on training set is 0.9
R2 score on test set is 0.873
Successfully store the trained model in dt.pkl
Saving figure dt_pattern1<jupyter_text>1.5 Extra Tree<jupyter_code># get feature data set and label data set, append a new feature column to X_all_15
X_all_15=df1_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
X_all_15['Na/Mg'] = X_all_15['Na'] / X_all_15['Mg']
y_all_15 =df1_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_15 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=13)
# divide augmented X and y into training and testing data
for train_index, test_index in split_15.split(df1_edit, df1_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_15 = X_all_15.loc[train_index]
    X_test_15 = X_all_15.loc[test_index] 
    y_train_15 = y_all_15.loc[train_index]
    y_test_15 = y_all_15.loc[test_index]
from sklearn.ensemble import ExtraTreesRegressor

# build up Extra Tree model
et_15 = ExtraTreesRegressor(n_estimators=500,bootstrap=False, oob_score=False,
                             max_leaf_nodes=20, random_state=42, n_jobs=-1)

# train the model and get the corresponding score
y_train_predict_15, y_test_predict_15, train_rmse_15, test_rmse_15, train_r2_15, test_r2_15 = \
score(et_15, 'Extra Tree', 'et', X_train_15, X_test_15, y_train_15, y_test_15)

# plot binary diagram
binary_plot(y_train = y_train_15,
            y_train_label = y_train_predict_15,
            y_test = y_test_15,
            y_test_label = y_test_predict_15,
            train_rmse = train_rmse_15,
            test_rmse = test_rmse_15,
            train_r2 = train_r2_15,
            test_r2 = test_r2_15)
save_fig("et_pattern1")<jupyter_output>----------Extra Tree----------
The RMSE on training set is 0.058
The RMSE on test set is 0.078
R2 score on training set is 0.932
R2 score on test set is 0.878
Successfully store the trained model in et.pkl
Saving figure et_pattern1<jupyter_text>1.6 Random Forest<jupyter_code># get feature data set and label data set, append a new feature column to X_all_16
X_all_16=df1_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
X_all_16['Na/Mg'] = X_all_16['Na'] / X_all_16['Mg']
y_all_16 =df1_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_16 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=13)
# divide augmented X and y into training and testing data
for train_index, test_index in split_16.split(df1_edit, df1_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_16 = X_all_16.loc[train_index]
    X_test_16 = X_all_16.loc[test_index] 
    y_train_16 = y_all_16.loc[train_index]
    y_test_16 = y_all_16.loc[test_index]
from sklearn.ensemble import RandomForestRegressor

# build up Random Forest model
rf_16 = RandomForestRegressor(n_estimators=500, oob_score=True, max_leaf_nodes=15,
                                n_jobs=-1, random_state=42)

# train the model and get the corresponding score
y_train_predict_16, y_test_predict_16, train_rmse_16, test_rmse_16, train_r2_16, test_r2_16 = \
score(rf_16, 'Random Forest', 'rf', X_train_16, X_test_16, y_train_16, y_test_16)

# plot binary diagram
binary_plot(y_train = y_train_16,
            y_train_label = y_train_predict_16,
            y_test = y_test_16,
            y_test_label = y_test_predict_16,
            train_rmse = train_rmse_16,
            test_rmse = test_rmse_16,
            train_r2 = train_r2_16,
            test_r2 = test_r2_16)
save_fig("rf_pattern1")<jupyter_output>----------Random Forest----------
The RMSE on training set is 0.055
The RMSE on test set is 0.069
R2 score on training set is 0.938
R2 score on test set is 0.907
Successfully store the trained model in rf.pkl
Saving figure rf_pattern1<jupyter_text>Pattern II<jupyter_code># read the excel of the data for Pattern II
data2 = os.path.join(DATA_PATH, "pattern2_data.xlsx")
df2 = pd.read_excel(data2, engine="openpyxl")
df2_edit = df2
df2_edit<jupyter_output><empty_output><jupyter_text>2.1 Linear Regression<jupyter_code># get feature data set and label data set
X_all_21 = df2_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_21 = df2_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_21 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
# divide augmented X and y into training and testing data
for train_index, test_index in split_21.split(df2_edit, df2_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_21 = X_all_21.loc[train_index]
    X_test_21 = X_all_21.loc[test_index] 
    y_train_21 = y_all_21.loc[train_index]
    y_test_21 = y_all_21.loc[test_index]
from sklearn.linear_model import LinearRegression

# Based on linear regression, bulid the polynomial regression with the augmented X 
linreg_21 = LinearRegression()
# train the model and get the corresponding score
y_train_predict_21, y_test_predict_21, train_rmse_21, test_rmse_21, train_r2_21, test_r2_21 = \
score(linreg_21, 'linear regression', 'Linear_new',X_train_21, X_test_21, y_train_21, y_test_21)
# plot binary diagram
binary_plot(y_train = y_train_21,
            y_train_label = y_train_predict_21,
            y_test = y_test_21,
            y_test_label = y_test_predict_21,
            train_rmse = train_rmse_21,
            test_rmse = test_rmse_21,
            train_r2 = train_r2_21,
            test_r2 = test_r2_21)
save_fig("lin_reg_pattern2")<jupyter_output>----------linear regression----------
The RMSE on training set is 0.068
The RMSE on test set is 0.078
R2 score on training set is 0.944
R2 score on test set is 0.918
Successfully store the trained model in Linear_new.pkl
Saving figure lin_reg_pattern2<jupyter_text>2.2 Polynomial Regression<jupyter_code>from sklearn.decomposition import PCA

# get feature data set and label data set
X_all_22 = df2_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_22 = df2_edit["Fe3+/Fetot(Correction)"].copy()

# do dimension reduction on the training feature data set 
pca_22 = PCA(n_components=6)
# the original dimensions 9 will be transformed into 6
X_all_reduced_22 = pca_22.fit_transform(X_all_22)
X_all_reduced_22.shape
pd.DataFrame(X_all_reduced_22)
from sklearn.preprocessing import PolynomialFeatures

# based on the reduction data, do dimension augmentation with the degree of the polynomial features 3
poly_features_22 = PolynomialFeatures(degree=3, include_bias=False)
X_all_augmented_22 = poly_features_22.fit_transform(X_all_reduced_22)
X_all_augmented_22.shape
# the columns name of the augmented data set
columns_name_22 = poly_features_22.get_feature_names()

# the augmentd data set, x0 refers to the first columns in the reduced data set
X_all_augmented_22 = pd.DataFrame(X_all_augmented_22)
X_all_augmented_22.columns = columns_name_22
X_all_augmented_22
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_22 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
# divide augmented X and y into training and testing data
for train_index, test_index in split_22.split(df2_edit, df2_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_22 = X_all_augmented_22.loc[train_index]
    X_test_22 = X_all_augmented_22.loc[test_index] 
    y_train_22 = y_all_22.loc[train_index]
    y_test_22 = y_all_22.loc[test_index]
from sklearn.linear_model import LinearRegression

# Based on linear regression, bulid the polynomial regression with the augmented X 
polyreg_22 = ElasticNet(alpha=0.0001, l1_ratio=0.0001, max_iter=100000, tol=0.0005)
# train the model and get the corresponding score
y_train_predict_22, y_test_predict_22, train_rmse_22, test_rmse_22, train_r2_22, test_r2_22 = \
score(polyreg_22, 'polynomial regression', 'poly_new',X_train_22, X_test_22, y_train_22, y_test_22)
# plot binary diagram
binary_plot(y_train = y_train_22,
            y_train_label = y_train_predict_22,
            y_test = y_test_22,
            y_test_label = y_test_predict_22,
            train_rmse = train_rmse_22,
            test_rmse = test_rmse_22,
            train_r2 = train_r2_22,
            test_r2 = test_r2_22)
save_fig("poly_reg_pattern2")<jupyter_output>----------polynomial regression----------
The RMSE on training set is 0.058
The RMSE on test set is 0.057
R2 score on training set is 0.96
R2 score on test set is 0.956
Successfully store the trained model in poly_new.pkl
Saving figure poly_reg_pattern2<jupyter_text>2.3 Artificial Neural Network<jupyter_code># get feature data set and label data set
X_all_23 = df2_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_23 = df2_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_23 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
# divide augmented X and y into training and testing data
for train_index, test_index in split_23.split(df2_edit, df2_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_23 = X_all_23.loc[train_index]
    X_test_23 = X_all_23.loc[test_index] 
    y_train_23 = y_all_23.loc[train_index]
    y_test_23 = y_all_23.loc[test_index]
from sklearn.neural_network import MLPRegressor

# build up Artificial Neural Network model
mlp_23 = MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', early_stopping=False,
                   hidden_layer_sizes=(20, 40, 40, 20), max_iter=2000, solver='adam', tol=1e-05,
                   random_state=9)

# train the model and get the corresponding score
y_train_predict_23, y_test_predict_23, train_rmse_23, test_rmse_23, train_r2_23, test_r2_23 = \
score(mlp_23, 'Artificial Neural Network', 'mlp_new', X_train_23, X_test_23, y_train_23, y_test_23)

# plot binary diagram
binary_plot(y_train = y_train_23,
            y_train_label = y_train_predict_23,
            y_test = y_test_23,
            y_test_label = y_test_predict_23,
            train_rmse = train_rmse_23,
            test_rmse = test_rmse_23,
            train_r2 = train_r2_23,
            test_r2 = test_r2_23)
save_fig("ann_pattern2")<jupyter_output>----------Artificial Neural Network----------
The RMSE on training set is 0.076
The RMSE on test set is 0.076
R2 score on training set is 0.93
R2 score on test set is 0.921
Successfully store the trained model in mlp_new.pkl
Saving figure ann_pattern2<jupyter_text>2.4 Artificial Neural Network Ensemble<jupyter_code># get feature data set and label data set
X_all_24 = df2_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_24 = df2_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_24 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
# divide augmented X and y into training and testing data
for train_index, test_index in split_24.split(df2_edit, df2_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_24 = X_all_24.loc[train_index]
    X_test_24 = X_all_24.loc[test_index] 
    y_train_24 = y_all_24.loc[train_index]
    y_test_24 = y_all_24.loc[test_index]
from sklearn.ensemble import BaggingRegressor

# single ANN model with the same hyperparameter as 2.3 ANN
mlp_24 = MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', early_stopping=False,
                   hidden_layer_sizes=(20, 40, 40, 20), max_iter=2000, solver='adam', tol=1e-05,
                   random_state=9)

# Use Bagging method to ensemble 500 ANN models
mlp_bag_24 = BaggingRegressor(mlp_24, n_estimators=500, bootstrap=True, bootstrap_features=True,
                              n_jobs=-1, oob_score=True, random_state=42)
# train the model and get the corresponding score
y_train_predict_24, y_test_predict_24, train_rmse_24, test_rmse_24, train_r2_24, test_r2_24 = \
score(mlp_bag_24, 'Artificial Neural Network Ensemble', 'mlp_bag_new', X_train_24, X_test_24, y_train_24, y_test_24)

# plot binary diagram
binary_plot(y_train = y_train_24,
            y_train_label = y_train_predict_24,
            y_test = y_test_24,
            y_test_label = y_test_predict_24,
            train_rmse = train_rmse_24,
            test_rmse = test_rmse_24,
            train_r2 = train_r2_24,
            test_r2 = test_r2_24)
save_fig("ann_ensemble_pattern2")<jupyter_output>----------Artificial Neural Network Ensemble----------
The RMSE on training set is 0.072
The RMSE on test set is 0.074
R2 score on training set is 0.939
R2 score on test set is 0.926
Successfully store the trained model in mlp_bag_new.pkl
Saving figure ann_ensemble_pattern2<jupyter_text>2.5 Decision Tree<jupyter_code># get feature data set and label data set, append a new feature column to X_all_25
X_all_25=df2_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
X_all_25['Na/Mg'] = X_all_25['Na'] / X_all_25['Mg']
y_all_25 =df2_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_25 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
# divide augmented X and y into training and testing data
for train_index, test_index in split_25.split(df2_edit, df2_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_25 = X_all_25.loc[train_index]
    X_test_25 = X_all_25.loc[test_index] 
    y_train_25 = y_all_25.loc[train_index]
    y_test_25 = y_all_25.loc[test_index]
from sklearn.tree import DecisionTreeRegressor

# build up Decision Tree model
dt_25 = DecisionTreeRegressor(ccp_alpha=0.0, criterion='mse', max_depth=5,
                              max_features=None, max_leaf_nodes=None, min_impurity_decrease=0.0, 
                              min_impurity_split=None, min_samples_leaf=1, min_samples_split=2,
                              min_weight_fraction_leaf=0.0, random_state=42, splitter='best',
                              presort='deprecated')

# train the model and get the corresponding score
y_train_predict_25, y_test_predict_25, train_rmse_25, test_rmse_25, train_r2_25, test_r2_25 = \
score(dt_25, 'Decision Tree', 'dt_new', X_train_25, X_test_25, y_train_25, y_test_25)

# plot binary diagram
binary_plot(y_train = y_train_25,
            y_train_label = y_train_predict_25,
            y_test = y_test_25,
            y_test_label = y_test_predict_25,
            train_rmse = train_rmse_25,
            test_rmse = test_rmse_25,
            train_r2 = train_r2_25,
            test_r2 = test_r2_25)
save_fig("dt_pattern2")<jupyter_output>----------Decision Tree----------
The RMSE on training set is 0.046
The RMSE on test set is 0.068
R2 score on training set is 0.975
R2 score on test set is 0.936
Successfully store the trained model in dt_new.pkl
Saving figure dt_pattern2<jupyter_text>2.6 Extra Tree<jupyter_code># get feature data set and label data set, append a new feature column to X_all_26
X_all_26=df2_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
X_all_26['Na/Mg'] = X_all_26['Na'] / X_all_26['Mg']
y_all_26 =df2_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_26 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
# divide augmented X and y into training and testing data
for train_index, test_index in split_26.split(df2_edit, df2_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_26 = X_all_26.loc[train_index]
    X_test_26 = X_all_26.loc[test_index] 
    y_train_26 = y_all_26.loc[train_index]
    y_test_26 = y_all_26.loc[test_index]
from sklearn.ensemble import ExtraTreesRegressor

# build up Extra Tree model
et_26 = ExtraTreesRegressor(n_estimators=500,bootstrap=False, oob_score=False,
                            max_leaf_nodes=15, random_state=42, n_jobs=-1)

# train the model and get the corresponding score
y_train_predict_26, y_test_predict_26, train_rmse_26, test_rmse_26, train_r2_26, test_r2_26 = \
score(et_26, 'Extra Tree', 'et_new', X_train_26, X_test_26, y_train_26, y_test_26)

# plot binary diagram
binary_plot(y_train = y_train_26,
            y_train_label = y_train_predict_26,
            y_test = y_test_26,
            y_test_label = y_test_predict_26,
            train_rmse = train_rmse_26,
            test_rmse = test_rmse_26,
            train_r2 = train_r2_26,
            test_r2 = test_r2_26)
save_fig("et_pattern2")<jupyter_output>----------Extra Tree----------
The RMSE on training set is 0.046
The RMSE on test set is 0.051
R2 score on training set is 0.974
R2 score on test set is 0.965
Successfully store the trained model in et_new.pkl
Saving figure et_pattern2<jupyter_text>2.7 Random Forest<jupyter_code># get feature data set and label data set, append a new feature column to X_all_27
X_all_27 = df2_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
X_all_27['Na/Mg'] = X_all_27['Na'] / X_all_27['Mg']
y_all_27 = df2_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_27 = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
# divide augmented X and y into training and testing data
for train_index, test_index in split_27.split(df2_edit, df2_edit["clustering_label"]):
    # train : test = 4 : 1
    X_train_27 = X_all_27.loc[train_index]
    X_test_27 = X_all_27.loc[test_index] 
    y_train_27 = y_all_27.loc[train_index]
    y_test_27 = y_all_27.loc[test_index]
from sklearn.ensemble import RandomForestRegressor

# build up Random Forest model
rnd_27 = RandomForestRegressor(n_estimators=500, oob_score=True, max_leaf_nodes=15,
                               n_jobs=-1, random_state=42)

# train the model and get the corresponding score
y_train_predict_27, y_test_predict_27, train_rmse_27, test_rmse_27, train_r2_27, test_r2_27 = \
score(rnd_27, 'Random Forest', 'rf_new', X_train_27, X_test_27, y_train_27, y_test_27)

# plot binary diagram
binary_plot(y_train = y_train_27,
            y_train_label = y_train_predict_27,
            y_test = y_test_27,
            y_test_label = y_test_predict_27,
            train_rmse = train_rmse_27,
            test_rmse = test_rmse_27,
            train_r2 = train_r2_27,
            test_r2 = test_r2_27)
save_fig("rf_pattern2")<jupyter_output>----------Random Forest----------
The RMSE on training set is 0.039
The RMSE on test set is 0.055
R2 score on training set is 0.982
R2 score on test set is 0.959
Successfully store the trained model in rf_new.pkl
Saving figure rf_pattern2<jupyter_text>Pattern III<jupyter_code># read the excel of the data for Pattern III
data3 = os.path.join(DATA_PATH, "pattern3_data.xlsx")
df3 = pd.read_excel(data3, engine="openpyxl")
df3_edit = df3
df3_edit<jupyter_output><empty_output><jupyter_text>3.1 Linear Regression<jupyter_code># get feature data set and label data set
X_all_31 = df3_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_31 = df3_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_31 = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=19)
# divide augmented X and y into training and testing data
for train_index, test_index in split_31.split(df3_edit, df3_edit["clustering_label"]):
    # train : test = 9 : 1
    X_train_31 = X_all_31.loc[train_index]
    X_test_31 = X_all_31.loc[test_index] 
    y_train_31 = y_all_31.loc[train_index]
    y_test_31 = y_all_31.loc[test_index]
from sklearn.linear_model import LinearRegression

# Based on linear regression, bulid the polynomial regression with the augmented X 
linreg_31 = LinearRegression()
# train the model and get the corresponding score
y_train_predict_31, y_test_predict_31, train_rmse_31, test_rmse_31, train_r2_31, test_r2_31 = \
score(linreg_31, 'linear regression', 'Linear_spl',X_train_31, X_test_31, y_train_31, y_test_31)
# plot binary diagram
binary_plot(y_train = y_train_31,
            y_train_label = y_train_predict_31,
            y_test = y_test_31,
            y_test_label = y_test_predict_31,
            train_rmse = train_rmse_31,
            test_rmse = test_rmse_31,
            train_r2 = train_r2_31,
            test_r2 = test_r2_31)
save_fig("lin_reg_pattern3")<jupyter_output>----------linear regression----------
The RMSE on training set is 0.049
The RMSE on test set is 0.04
R2 score on training set is 0.385
R2 score on test set is 0.574
Successfully store the trained model in Linear_spl.pkl
Saving figure lin_reg_pattern3<jupyter_text>3.2 Extra Tree<jupyter_code># get feature data set and label data set, append a new feature column to X_all_32
X_all_32=df3_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_32 =df3_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_32 = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=19)
# divide augmented X and y into training and testing data
for train_index, test_index in split_32.split(df3_edit, df3_edit["clustering_label"]):
    # train : test = 9 : 1
    X_train_32 = X_all_32.loc[train_index]
    X_test_32 = X_all_32.loc[test_index] 
    y_train_32 = y_all_32.loc[train_index]
    y_test_32 = y_all_32.loc[test_index]
from sklearn.ensemble import ExtraTreesRegressor

# build up Extra Tree model
etr_clf_32 = ExtraTreesRegressor(n_estimators=500,bootstrap=False, oob_score=False,
                                 max_leaf_nodes=15, random_state=42, n_jobs=-1)

# train the model and get the corresponding score
y_train_predict_32, y_test_predict_32, train_rmse_32, test_rmse_32, train_r2_32, test_r2_32 = \
score(etr_clf_32, 'Extra Tree', 'et_spl', X_train_32, X_test_32, y_train_32, y_test_32)

# plot binary diagram
binary_plot(y_train = y_train_32,
            y_train_label = y_train_predict_32,
            y_test = y_test_32,
            y_test_label = y_test_predict_32,
            train_rmse = train_rmse_32,
            test_rmse = test_rmse_32,
            train_r2 = train_r2_32,
            test_r2 = test_r2_32)
save_fig("et_pattern3")<jupyter_output>----------Extra Tree----------
The RMSE on training set is 0.031
The RMSE on test set is 0.034
R2 score on training set is 0.765
R2 score on test set is 0.697
Successfully store the trained model in et_spl.pkl
Saving figure et_pattern3<jupyter_text>3.3 Random Forest<jupyter_code># get feature data set and label data set, append a new feature column to X_all_33
X_all_33 = df3_edit.drop(["Fe3+/Fetot(Correction)", "clustering_label"], axis=1)
y_all_33 = df3_edit["Fe3+/Fetot(Correction)"].copy()
from sklearn.model_selection import StratifiedShuffleSplit

# stratified random sampling based on the results of KMeans Clustering
split_33 = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=19)
# divide augmented X and y into training and testing data
for train_index, test_index in split_33.split(df3_edit, df3_edit["clustering_label"]):
    # train : test = 9 : 1
    X_train_33 = X_all_33.loc[train_index]
    X_test_33 = X_all_33.loc[test_index] 
    y_train_33 = y_all_33.loc[train_index]
    y_test_33 = y_all_33.loc[test_index]
from sklearn.ensemble import RandomForestRegressor

# build up Random Forest model
rnd_clf_33 = RandomForestRegressor(n_estimators=500, oob_score=True, max_leaf_nodes=15,
                                n_jobs=-1, random_state=42)

# train the model and get the corresponding score
y_train_predict_33, y_test_predict_33, train_rmse_33, test_rmse_33, train_r2_33, test_r2_33 = \
score(rnd_clf_33, 'Random Forest', 'rf_spl', X_train_33, X_test_33, y_train_33, y_test_33)

# plot binary diagram
binary_plot(y_train = y_train_33,
            y_train_label = y_train_predict_33,
            y_test = y_test_33,
            y_test_label = y_test_predict_33,
            train_rmse = train_rmse_33,
            test_rmse = test_rmse_33,
            train_r2 = train_r2_33,
            test_r2 = test_r2_33)
save_fig("rf_pattern3")<jupyter_output>----------Random Forest----------
The RMSE on training set is 0.025
The RMSE on test set is 0.024
R2 score on training set is 0.84
R2 score on test set is 0.846
Successfully store the trained model in rf_spl.pkl
Saving figure rf_pattern3
StarcoderdataJupyterStructuredCleanDedup