# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %matplotlib inline
import pandas as pd
import numpy as np

# # Intro to Pandas

# There are primarily two data structures
# 1. Series
# 2. DataFrame

# ## 1. Series

# - An array of elements with types similar to Numpy
# - Contains an index for each element

# ### 1.1 Create Series

series_a =  pd.Series([3, 1, 2.3, 'a', True])

series_a

series_a.values # list all the values in a series

series_a.index # list the index values

# ### 1.2 Let us create a series with int objects

series_b = pd.Series([1, 4, 2])

series_b

# ### 1.3 Access any element with index

series_b[1] # access the element at position 2 (default starting index is 0)

# ### 1.4 Access multiple elements

# +
# x:y:z --> access every zth element starting from index x upto index y 
# if x is missing --> consider 0
# if z is missing --> consider all elements

series_b[:2] # access all elements from 0 upto 2 (2 not included) i.e., 0 and 1
# -

# ### 1.5 Edit any element using index

series_b[1] = 3

series_b

# ### 1.6 Query And Subset

series_b > 1

series_b[series_b > 1]

# ### 1.7 Edit multiple elements

series_b.loc[series_b>1] = [2, 3]

series_b

# ### 1.8 Perform operations

series_b + 10

series_b ** 2

np.log(series_b)

# ### 1.9 Identify null values

series_c = pd.Series([np.nan, 2, 23, 4])

series_c

pd.isnull(series_c)

pd.notnull(series_c)

# ### 1.10 Operate two series of equal length

series_1 = pd.Series([1, 2, 3])
series_2 = pd.Series([4, 5, 6])

series_1 + series_2

series_1 * series_2

# ### 1.11 Operate two series of unequal length

series_1 = pd.Series([1, 2, 3])
series_2 = pd.Series([4, 5, 6, 7])

series_1 + series_2

# ## 2. DataFrame

# - A collection of Series where each Series represents one Column
# - Similar to data represented in Excel/Database tables

# ### 2.1 Create DataFrame

df_dict = {"a": [1, 2, 3], "b": ['a', 'b', 'c'], "c": [True, False, True]}

df = pd.DataFrame(df_dict)

df

df_dict = [[1, 2, 3], ['a', 'b', 'c'], [True, False, True]]
df = pd.DataFrame(df_dict, columns = ['a', 'b', 'c'], index=[1, 2, 3])

df

# ### 2.2 Reading and Writing

df = pd.read_csv("../datasets/Oscars-demographics-DFE.csv", encoding='latin1')

df.head()

# ### 2.3 Subset rows

len(df)

df.index = range(100, 541)

df.loc[100:105] # based on index names

df.iloc[0:5] # based on actual position of rows

df.head(10)

df.tail(3)

df.sample(n=10)

df.sample(frac=0.01)

df[df['date_of_birth:confidence']==0.6667]

# ### 2.4 Subset Columns

# #### 2.4.1 Single Column

df['_golden'] # pick one column - returns Series

df._golden

# #### 2.4.2 Multiple Columns

df[['_unit_id', '_golden', '_unit_state']] # pick multiple columns - returns DataFrame

df.filter(regex='date_of_birth')

# ### 2.5 Subset Rows and Columns

# the first condition is on rows, the second condition is on columns
df.loc[ df['date_of_birth:confidence']==1 , ['_unit_id', '_golden'] ]

# ## 2.6 Dropping rows, columns

# dropping columns
df.drop(['birthplace_gold', 'race_ethnicity'], axis=1)

# dropping duplicate rows
df.drop_duplicates()

# ## 2.7 Summarize Data

# Numeric columns - calculates descriptive statistics
df.describe()

# Categorical columns - the unique values with their count of appearance in that column
df['_unit_state'].value_counts()

# the count of unique elements in a column
df['_unit_state'].nunique()

# Calculate the length of the DataFrame
len(df)

# +
# apply a function to summarize

# in-built functions - max, min, sum, count, median, mean, std, var
df.max() # max


# +
# apply your custom function
def func(x):
    return x+1

df[['year_of_award', 'year_of_award:confidence']].apply(func)
# -

# ## 2.8 Group Data

# +
# group by a set of columns and apply functions

# apply in-built functions directly

df.groupby(['year_of_award']).sum()


# +
# apply custom functions using agg
def custom_mean(x):
    return np.mean(x)+1

df.groupby(['year_of_award'])['_golden', '_trusted_judgments'].agg(custom_mean)
# -

# ## 2.9 Missing values

# drop rows with any columns having NA
df.dropna()

# can apply on a column
df['_last_judgment_at'].dropna()

# ### 2.9.1 Replace missing values

# Replace missing values specific to each column
df.loc[df['_last_judgment_at'].isna(), '_last_judgment_at']

df.loc[df['_last_judgment_at'].isna(), '_last_judgment_at'] = df['_last_judgment_at'].mode()[0]

# use the boolean index to replace the NAs with a custom value
df[['birthplace_gold', 'race_ethnicity_gold']].fillna('Australia')

# ## 2.10 Create new rows / columns

simple_df = pd.DataFrame({1:[1, 2, 3], 2:['a', 'b', 'c'], 3:[True, False, True]})

simple_df

# ### 2.10.1 Create new row using .loc

# add the new row using a new index value (3 here)
simple_df.loc[3] = [4, 'd', False]

simple_df

# ### 2.10.2 Create new column

simple_df

# the new column should be of the same length as the df
simple_df[4] = [42, 2.3, 3, 65]

simple_df

# ## 2.11 Combining two dataframes

# ### 2.11.1 Create new row by combining DataFrames

new_df = pd.DataFrame([[4, 'd', False]], columns=[1,2,3])

new_df

# Concat function can be used for concatenating rows/columns
# if axis = 0 -> rows will be concatenated
# if axis = 1 -> columns will be concatenated
pd.concat([simple_df, new_df], axis=0)

# ### 2.11.2 Merge

# Merge is like the join operation from SQL.
#
# You will merge two dataframes based on a column common between the two.
#
# Wherever the values of the column match in both dataframes, a new row which is a concat of both rows from A and B is formed

sales_df = pd.DataFrame([[1, 23, 100], [1, 45, 142], [2, 52, 45]], columns = ['customer_id', 'order_id', 'order_value'])
sales_df

customer_data = pd.DataFrame([[1, 'Male', 25], [2, 'Female', 49], [3, 'Female', 32]], columns = ['customer_id', 'gender', 'age'])

customer_data

# merge with a left condition - suggesting us to keep all rows from left dataframe
pd.merge(sales_df, customer_data, on="customer_id", how="left")

# merge with a right condition - suggesting us to keep all rows from right dataframe
pd.merge(sales_df, customer_data, on="customer_id", how="right")

# ## 2.12 Reshaping your DataFrame

# ### 2.12.1 Sorting

# sort_values takes columns
df.sort_values(['_trusted_judgments'], ascending=False)

# ### 2.12.2 Reset index

df.reset_index()

# ### 2.12.3 Pandas Melt

simple_df

# keeps the columns in id_vars intact, takes columns in value_vars and creates a row for each entry
pd.melt(simple_df, id_vars=[1], value_vars=[2, 3])

# ### 2.12.4 Pandas Pivot

simple_df

# +
# index columns get the index in the pivot output
# columns in column would be the columsn of the pivot output
# values represent the value that will be placed in the dataframe
# aggfunc can be used when there are multiple values in that position
# fill_value fills NaN with the argument value

pd.pivot_table(simple_df, index=[1], values=[4], columns=[2], aggfunc=np.mean, fill_value=0)
# -

# ## 2.13 Plotting

df['birthplace:confidence'].plot.hist()

df.plot.scatter(x="_trusted_judgments", y="birthplace:confidence")



StarcoderdataJupyterScriptsDedupFiltered