# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Exploratory Data Analyses

# Goal:
# Predict **Circulation Loss** based in Geology, Geophysics, Drilling and Reservour Engineering Studies in planning phase with the objective to choose between MPD and conventional rigs to drill the well.

# %autosave 0

# %matplotlib inline
import itertools
import sqlite3
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pylab as plt
from pandas.plotting import scatter_matrix
#from customplot import *

# define tamanho padrão das figuras
#plt.rc('figure', figsize=(10, 5))
#fizsize_with_subplots = (10, 100)
bin_size = 10 

dfbase = pd.read_csv('baseProjeto_to_EDA.csv', index_col=0)
dfbase.head(5)

categorize = lambda x: x.astype('category')
cat = [ 'MGP1', 'MGP2', 'MGP3', 'MGP4', 'MGP5', 'MGP6', 'MGP7', 'MGP8', 
       'MGP9', 'MGP10', 'MGP11', 'MGP12', 'MGP13', 'MGP14','Perda30', 
       'PerdaAPI','PerdaPS']
dfbase[cat] = dfbase[cat].apply(categorize,axis=0)

# ## *Algumas estatísticas dos dados*
# *apenas para dados numéricos

dfbase.describe()

# The max value in `ATRIB_DIST1` look a outlier

print(dfbase.sort_values('ATRIB_DIST1', ascending=False).head(5)['ATRIB_DIST1'])

# In this case, I am goint to substitute it using the **75%** value

dfbase.loc[dfbase['ATRIB_DIST1']==3000, 'ATRIB_DIST1'] = 1000

print(dfbase.sort_values('ATRIB_DIST1', ascending=False).head(5)['ATRIB_DIST1'])

dfbase.info()

dfbase.corr(method='pearson', min_periods=1)

plt.matshow(dfbase.corr(method='pearson', min_periods=1))

# ## *Removendo features com alta correlação*

# +
# Create correlation matrix
corr_matrix = dfbase.corr().abs()

# Select upper triangle of correlation matrix
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))

# Find index of feature columns with correlation greater than 0.95
to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]
# -

to_drop

# Drop correlatede features 
#dfbase=dfbase.drop(to_drop, axis=1)
# Adjust y position
dftemp = dfbase[['Perda30','PerdaAPI','PerdaPS','PERDA_MAX']]
dfbase=dfbase.drop(['Perda30','PerdaAPI','PerdaPS','PERDA_MAX'], axis=1)
dfbase[['Perda30','PerdaAPI','PerdaPS','PERDA_MAX']]=dftemp[['Perda30','PerdaAPI','PerdaPS','PERDA_MAX']]

dfbase.head()

dfbase.corr(method='pearson', min_periods=1).columns

# +
plt.rc('figure', figsize=(20, 20))
#fizsize_with_subplots = (100, 200)
#alpha = dfbase.corr(method='pearson', min_periods=1).columns

fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(dfbase.corr(method='pearson', min_periods=1))
fig.colorbar(cax)

#ax.set_xticklabels(alpha)
#ax.set_yticklabels(alpha)

plt.show()

# -

# ## *Verificando outliers*

x=dfbase.describe().columns # Obtaining the columns

# +
#dfbase.info()
# -

numbers = ['ATRIB_MED1', 'ATRIB_MAX1', 'ATRIB_DIST1', 'DIFP']
categorics = ['MGP1', 'MGP2',
       'MGP3', 'MGP4', 'MGP5', 'MGP6', 'MGP7', 'MGP8', 'MGP9', 'MGP10',
       'MGP11', 'MGP12', 'MGP13', 'MGP14']
y = ['Perda30']

fig_size=plt.rcParams["figure.figsize"]
fig_size[0] = 12
fig_size[1] = 10
plt.rcParams["figure.figsize"] = fig_size
plt.figure(1)
a=1
for i in numbers:
    plt.subplot(len(x),1,a) #subplot(nrows, ncols, index, **kwargs)
    sns.swarmplot(x='Perda30', y=i, data=dfbase)
    a+=1
plt.show()


fig_size[0] = 12
fig_size[1] = 5
plt.rcParams["figure.figsize"] = fig_size
for i in numbers:
    dfbase.boxplot(column=i, by='Perda30')

for i in numbers:
    dfbase.hist(column=i)

# +
# ax1 = dfbase.plot.scatter(x='ATRIB_MED1', y='PERDA_MAX', c='DarkBlue')
# -

sns.pairplot(x_vars=['ATRIB_MED1'], y_vars=['PERDA_MAX'], data=dfbase, hue="Perda30", size=5)

# +
# ax1 = dfbase.plot.scatter(x='ATRIB_MAX1', y='PERDA_MAX', c='DarkBlue')
# -

sns.pairplot(x_vars=['ATRIB_MAX1'], y_vars=['PERDA_MAX'], data=dfbase, hue="Perda30", size=5)

# +
#ax1 = dfbase.plot.scatter(x='ATRIB_DIST1', y='PERDA_MAX', c='DarkBlue')
# -

sns.pairplot(x_vars=['ATRIB_DIST1'], y_vars=['PERDA_MAX'], data=dfbase, hue="Perda30", size=5)

# +
# ax1 = dfbase.plot.scatter(x='DIFP', y='PERDA_MAX', c='DarkBlue')
# -

sns.pairplot(x_vars=['DIFP'], y_vars=['PERDA_MAX'], data=dfbase, hue="Perda30", size=5)

# +
#sns.pairplot(dfbase)
#plt.show()

# +
# dfbase.to_csv('baseProjeto_entradaModelo.csv', encoding='utf-8')
# -

count_class_0, count_class_1 = dfbase.Perda30.value_counts()

print(count_class_0, count_class_1)

dfbase.Perda30.value_counts().plot(kind='bar', title='Count (target)');

df_class_0 = dfbase[dfbase['Perda30'] == 0]
df_class_1 = dfbase[dfbase['Perda30'] == 1]

# +
df_class_1_over = df_class_1.sample(count_class_0, replace=True)
df_test_over = pd.concat([df_class_0, df_class_1_over], axis=0)

print('Random over-sampling:')
print(df_test_over.Perda30.value_counts())

df_test_over.Perda30.value_counts().plot(kind='bar', title='Count (target)');
# -

df_test_over.to_csv('baseProjeto_over.csv', encoding='utf-8')

StarcoderdataJupyterScriptsDedupFiltered