# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python3.5 Predict-poverty
#     language: python
#     name: predict-poverty
# ---

# # Create deep features dataset

from keras.applications.vgg16 import VGG16
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input
from keras.models import Model
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt

df_rwanda = pd.read_csv('./data/rwanda.csv')

df_rwanda.head()

base_model = VGG16(weights='imagenet')
model = Model(inputs=base_model.input, outputs=base_model.get_layer('fc2').output)

folder_path = './data/daytime_images/rwanda/'

images = []
for img in os.listdir(folder_path):
    img = os.path.join(folder_path, img)
    img = image.load_img(img, target_size=(224, 224))
    img = image.img_to_array(img)
    img = np.expand_dims(img, axis=0)
    images.append(img)
images = np.vstack(images)

images = np.vstack(images)

features = model.predict(images)

features.shape

features = list(features)

df_features = pd.DataFrame({'id': df_rwanda['cluster'].values,
                            'features': features})

df_features.head()

df_merged = pd.merge(df_rwanda, df_features, how='inner', left_on='cluster', right_on='id')[['cluster', 'latitude',
                                                                                            'longitude',
                                                                                            'wealth_index',
                                                                                            'mean_light',
                                                                                            'median_light',
                                                                                            'std_light',
                                                                                            'min_light',
                                                                                            'max_light',
                                                                                            'features']]
df_merged.head()

df_merged.to_pickle('./data/rwanda_deep.csv')

# # Predict from deep features

import pandas as pd
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import numpy as np

# +
df_rwanda = pd.read_pickle('./data/rwanda_deep.csv')

X = np.array([df_rwanda['features'].values[i] for i in range(df_rwanda['features'].values.shape[0])])
X = np.concatenate([X, df_rwanda['mean_light'].values.reshape((-1, 1))], axis=1)
y = df_rwanda['wealth_index'].values.reshape((-1, 1))

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.40, random_state=42)

# +
from sklearn.linear_model import LinearRegression

estimator = LinearRegression()
estimator.fit(X_train, y_train)

print('MSE on train data :', round(((y_train - estimator.predict(X_train)) ** 2).mean()))
print('MSE on test data :', round(((y_test - estimator.predict(X_test)) ** 2).mean(), 2))
print('R2-score :', round(estimator.score(X_test, y_test), 2))

# +
from sklearn.decomposition import PCA

pca = PCA(n_components=4)
pca.fit(X_train[:,:4096])

# +
X_train_sparse = pca.transform(X_train[:, :4096])
X_test_sparse = pca.transform(X_test[:, :4096])

X_train_sparse = np.concatenate([X_train_sparse, X_train[:,4096:]], axis=1)
X_test_sparse = np.concatenate([X_test_sparse, X_test[:,4096:]], axis=1)

print(X_train_sparse.shape, X_train.shape)

# +
estimator = LinearRegression()
estimator.fit(X_train_sparse, y_train)

print('MSE on train data :', round(((y_train - estimator.predict(X_train_sparse)) ** 2).mean(), 2))
print('MSE on test data :', round(((y_test - estimator.predict(X_test_sparse)) ** 2).mean(), 2))
print('R2-score :', round(estimator.score(X_test_sparse, y_test), 2))

# +
from sklearn.linear_model import Ridge, RidgeCV

estimator = RidgeCV(alphas=np.logspace(0, 6, 1000))
estimator.fit(X_train_sparse, y_train)

print('alpha = ', estimator.alpha_)

print('MSE on train data :', round(((y_train - estimator.predict(X_train_sparse)) ** 2).mean(), 2))
print('MSE on test data :', round(((y_test - estimator.predict(X_test_sparse)) ** 2).mean(), 2))
print('R2-score :', round(estimator.score(X_test_sparse, y_test), 2))

# +
from sklearn.linear_model import Lasso, LassoCV

estimator = LassoCV(alphas=np.logspace(-6, 7, 100))
estimator.fit(X_train_sparse, y_train)

print('alpha = ', estimator.alpha_)

print('MSE on train data :', round(((y_train - estimator.predict(X_train_sparse)) ** 2).mean(), 2))
print('MSE on test data :', round(((y_test - estimator.predict(X_test_sparse)) ** 2).mean(), 2))
print('R2-score :', round(estimator.score(X_test_sparse, y_test), 2))

# +
from sklearn.feature_selection import SelectFromModel

model = SelectFromModel(estimator, prefit=True)
X_new = model.transform(X_train_sparse)
X_new.shape

# +
from sklearn.preprocessing import PolynomialFeatures

poly = PolynomialFeatures(degree=1)
X_train_pol = poly.fit_transform(X_train_sparse)
X_test_pol = poly.fit_transform(X_test_sparse)

# +
from sklearn.linear_model import Ridge, RidgeCV

estimator = RidgeCV(alphas=np.logspace(0, 10, 1000))
estimator.fit(X_train_pol, y_train)

print('alpha = ', estimator.alpha_)

print('MSE on train data :', round(((y_train - estimator.predict(X_train_pol)) ** 2).mean(), 2))
print('MSE on test data :', round(((y_test - estimator.predict(X_test_pol)) ** 2).mean(), 2))
print('R2-score :', round(estimator.score(X_test_pol, y_test), 2))
# -



StarcoderdataJupyterScriptsDedupFiltered