# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Pandas-2 Continues ...

import pandas as pd

data = pd.read_csv('My Uber Drives - 2016.csv')

data.head()

# ## 5. Groupby Summarize

# For each start location, find the mean distance of the trip

data.groupby("START*")["MILES*"].agg(["mean"]).head()

# find the mean and total distance travelled
data.groupby("START*")["MILES*"].agg(["mean", "sum", "count"]).sort_values('count', ascending=False)

data["START*"].value_counts()

# ## Save DataFrame into CSV

data

data.to_csv("data_baru.csv")

# Saving df without index

data.to_csv("data_baru2.csv", index = False)

# ## Handling Missing Value

import numpy as np

new_df = pd.DataFrame({'col_a': [1,2,4,1, np.nan, np.nan, 5],
                       'col_b': [3,7, np.nan, 9, None, 5, 8],
                       'col_c': ['a', '?', 'x', 'y', '--', np.nan, 'r'],
                       'col_d': [True, True, np.nan, None, False, True, False]})

new_df

new_df['col_c'].unique()

new_df.to_csv('data_saya.csv', index=False)

# np.nan, None and NaT (for datetime64[ns] types) are standard missing value for Pandas.

# ### Find Missing Values

# Pandas provides `isnull()`, `isna()` functions to detect missing values. Both of them do the same thing.

new_df.shape

new_df.info()

new_df.isna()

# df.isna().any() returns a boolean value for each column. If there is at least one missing value in that column, the result is True.

new_df.isna().any()

new_df.isna().sum()

new_df.isnull().sum()

# Missing value can be irrevant characters, such as "?" and "--" character in col_c\
# These character can't be detected as missing value by Pandas

# If we know what kind of characters used as missing values in the dataset, we can handle them by creating the dataframe using `na_values` parameter:
#

missing_values = ["?", "--"]
df2 = pd.read_csv("data_saya.csv", na_values = missing_values)

df2.isna().sum()

df2

# Another option is to use pandas replace() function to handle these values after a dataframe is created:
#

df3 = new_df.replace({"?": np.nan, "--": np.nan})

df3.isna().sum()

df3

# Compare to original df

new_df.isna().sum()

# ## Drop missing value

df3

# We can drop a row or column with missing values using dropna() function. We can use some condition:\
# how='any' : drop if there is any missing value\
# how='all' : drop if all values are missing

df3.dropna(axis=0, how='all', inplace=True)
df3

df3.dropna(axis=0, how='any')

# We can use 'thresh' parameter to set a threshold for missing values in order for a row/column to be dropped. Thresh is the amount of non-na value

df3

df3.dropna(axis=0, thresh=3)

# ### Replacing missing values

# `fillna()` function in Pandas is used to replace missing values with another values.\
# Missing values can be replaced by:
# 1. Special value
# 2. Aggregate value, such as mean, median, etc

# #### Replacing with scalar

df3

df5 = df3.fillna(0)
df5

df3

df3['col_a'].mean()

df3

df3.iloc[:, ::-1]# [m:n:s] m : inclusive, n : exclusive , s : step (default: 1)

df3

df3.iloc[:, 0] = df3.iloc[:, 0].fillna(df3.iloc[:, 0].mean())
df3

dfx = new_df.replace({"?": np.nan, "--": np.nan})
dfx

dfx['col_a'] = dfx['col_a'].fillna(dfx['col_a'].mode()[0])
dfx

df7 = new_df.replace({"?": np.nan, "--": np.nan})
df7

df8 = new_df.replace({"?": np.nan, "--": np.nan})
df8

# Take the last seen values by using `ffill` (forward fill)

df8.fillna(method='ffill', inplace=True)

df8

df9 = new_df.replace({"?": np.nan, "--": np.nan})
df9

df9.fillna(method='bfill', inplace=True)

df9

# ---

# ## Assignment-2

# 1. Find how many missing values in each column of Titanic data

import pandas as pd

titanic = pd.read_csv('train.csv')

titanic.info()

titanic.isnull().sum()

# 2. Replace the missing values with the following values:\
#     -Embarked 'S'\
#     -Age 'mean'\
#     -Cabin 'mode'

titanic['Embarked'] = titanic['Embarked'].fillna('S')
titanic.isnull().sum()

titanic['Age'] = titanic['Age'].fillna(titanic['Age'].mean())
titanic.isnull().sum()

titanic['Cabin'] = titanic['Cabin'].fillna(titanic['Cabin'].mode()[0])
titanic.isnull().sum()

# 3. find how many passenger is survived in each class group by jenis kelamin nya

titanic.head()

titanic.groupby(['Pclass', 'Sex'])[['Survived']].sum()

titanic[titanic['Survived']==1].shape[0]

91+45+70+17+72+47

StarcoderdataJupyterScriptsDedupFiltered