# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

import pandas as pd
import numpy as np
import os

raw_data_path = os.path.join(os.path.pardir, 'data', 'raw')
train_file_path = os.path.join(raw_data_path, 'train.csv')
test_file_path = os.path.join(raw_data_path, 'test.csv')

train_df = pd.read_csv(train_file_path, index_col='PassengerId')
test_df = pd.read_csv(test_file_path, index_col='PassengerId')

test_df['Survived'] = -888

df = pd.concat((train_df, test_df), axis=0)

df[df.Embarked.isnull()]

pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Embarked)

df.groupby(['Pclass', 'Embarked']).Fare.median()

df.Embarked.fillna('C', inplace=True)

df.info()

median_fare = df.loc[(df.Pclass == 3) & (df.Embarked == 'S'), 'Fare'].median()
print median_fare

df.Fare.fillna(median_fare, inplace=True)

df.info()

df[df.Age.isnull()]

# %matplotlib inline
df.Age.plot(kind='hist', bins=20, color='c')

df.groupby('Sex').Age.median()

df[df.Age.notnull()].boxplot('Age', 'Sex')

df[df.Age.notnull()].boxplot('Age', 'Pclass')

df.Name


def get_title(name):
    first_name_with_title = name.split(',')[1]
    title = first_name_with_title.split('.')[0]
    title = title.strip().lower()
    return title


df.Name.map(lambda x: get_title(x))

df.Name.map(lambda x: get_title(x)).unique()


def get_title(name):
    title_group = {
        'mr' : 'Mr', 
        'mrs' : 'Mrs', 
        'miss' : 'Miss', 
        'master' : 'Master', 
        'don' : 'Ser', 
        'rev' : 'Ser', 
        'dr' : 'Officer', 
        'mme' : 'Mrs', 
        'ms' : 'Mrs',
        'major' : "Officer", 
        'lady' : 'Lady', 
        'sir' : 'Sir', 
        'mlle' : 'Miss', 
        'col' : 'Officer', 
        'capt' : 'Officer',
        'the countess' : 'Lady',
        'jonkheer' : 'Sir', 
        'dona' : 'Lady'
    }
    first_name_with_title = name.split(',')[1]
    title = first_name_with_title.split('.')[0]
    title = title.strip().lower()
    return title_group[title]


df['Title'] = df.Name.map(lambda x: get_title(x))

df.Title

df[df.Age.notnull()].boxplot('Age', 'Title')

title_age_median = df.groupby('Title').Age.transform('median')
title_age_median

df.Age.fillna(title_age_median, inplace=True)

df.info()

# # Outliners

df.Age.plot(kind='hist', bins=20, color='c')

df.loc[df.Age > 70]

df.Fare.plot(kind='box')

df.loc[df.Fare == df.Fare.max()]

LogFare = np.log(df.Fare + 1.0)

LogFare.plot(kind="hist", color="c", bins=20)

pd.qcut(df.Fare, 4)

pd.qcut(df.Fare, 4, labels=['very low', 'low', 'hight', 'very hight']).value_counts().plot(kind='bar', color='c', rot=0)

df['Fare_Bin'] = pd.qcut(df.Fare, 4, labels=['very_low', 'low', 'hight', 'very_hight'])

df.info()

# # Feature engineering

df['AgeState'] = np.where(df['Age'] >= 18, 'Adult', 'Child')

df['AgeState'].value_counts()

pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].AgeState)

df['FamilySize'] = df.Parch + df.SibSp + 1

df['FamilySize'].plot(kind='hist', color="c")

df.loc[df.FamilySize == df.FamilySize.max()]

pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].FamilySize)

df['IsMother'] = np.where(((df.Sex == 'female') & (df.Parch > 0) & (df.Age > 18) & (df.Title != 'Miss')), 1, 0)

pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].IsMother) # survied mother with children

# # Deck

df.Cabin

df.Cabin.unique()

df.loc[df.Cabin == 'T' ]

df.loc[df.Cabin =='T', 'Cabin'] = np.NaN

df.Cabin.unique()


def get_deck(cabin):
    return np.where(pd.notnull(cabin), str(cabin)[0].upper(), 'Z')
df['Deck'] = df['Cabin'].map(lambda x: get_deck(x))

df.Deck.value_counts()

pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Deck)

df.info()

# # Categorial function

df['IsMale'] = np.where(df.Sex == 'male', 1, 0)

df = pd.get_dummies(df, columns=['Deck', 'Pclass', 'Title', 'Fare_Bin', 'Embarked', 'AgeState'])

df.info()

# # Droping or reco

df.drop(['Cabin', 'Name', 'Ticket', 'Parch', 'SibSp', 'Sex'], axis=1, inplace=True)

columns = [column for column in df.columns if column != 'Survived']
columns = ['Survived'] + columns
df = df[columns]

df.info()

processed_data_path = os.path.join(os.path.pardir, 'data', 'processed')
write_train_path = os.path.join(processed_data_path, 'train.csv')
write_test_path = os.path.join(processed_data_path, 'test.csv')

df.loc[df.Survived != -888].to_csv(write_train_path)

columns = [column for column in df.columns if column != 'Survived']
df.loc[df.Survived == -888, columns].to_csv(write_test_path)



StarcoderdataJupyterScriptsDedupFiltered