# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Data Mining practice and reference
#
# The methods are applied on income dataset downloaded from:
#
# https://www.kaggle.com/ahmettezcantekin/beginner-datasets

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display
import seaborn as sns
import math
import missingno as msno # for missing value visualizations
from typing import List # for typing hints
import collections # for ordered dict
import itertools # for combinations

# ## Initial exploration

df = pd.read_csv('data/income.csv')
df.head()

education_dict = {x['education-num']:x['education'] for i, x in df.iterrows()}
education_dict_ordered = collections.OrderedDict(sorted(education_dict.items()))
education_dict_ordered

# + tags=[]
df.info()

# + [markdown] tags=[]
# ## Data Preprocessing
# -

# Casting to correct data types.

# +
df['workclass'] = df['workclass'].astype('category')
df['education'] = df['education'].astype('category')
df['marital-status'] = df['marital-status'].astype('category')
df['occupation'] = df['occupation'].astype('category')
df['relationship'] = df['relationship'].astype('category')
df['race'] = df['race'].astype('category')
df['sex'] = df['sex'].astype('category')
df['native-country'] = df['native-country'].astype('category')
df['income >50K'] = df['income >50K'].astype('category')

df.info()

# + tags=[]
df.describe()
# -



# ### Handling missing data

df.isnull().values.any()

s = df.isnull().sum()
s[s != 0] # columns with missing values and how many missing values they contain

len(df.index) # record count

s[s != 0] / len(df.index) # missing values relative to all values

# All columns containing missing values are categorical. The highest missing value column is the workclass column with around 5.6% of values missing. Work class and occupation show almost equal amount of null values, hence further exploration is carried out.

msno.matrix(df[s[s != 0].index])

# The above plot shows that values in columns work class and occupation are missing in similar places. Occupation has seven more values than work class. The occupation commonly determines one's work class; it can be therefore assumed that there is a strong correlation between the two columns; therefore, some missing values may be inferred. Since the missing values are around 5%, argument can be made for removing those values from the analysis.

df[(df['workclass'].notnull()) & (df['occupation'].isnull())][['workclass', 'occupation']]

# Further investigation shows, that **all Never-worked workclass has stated no occupation**, which makes sense. The categories take into account self-employment and enterprenurial optionsmissing values in occupation can be replaced by 'unemployed.'

try: 
    df['occupation'] = df['occupation'].cat.add_categories(['unemployed'])
    df.loc[(df['workclass'].notnull()) & (df['occupation'].isnull()), 'occupation'] = 'unemployed'
except ValueError as e:
    pass
df[(df['workclass'].notnull()) & (df['occupation'] == 'unemployed')] 

# The above table shows the result of replacement. All unemployed people put some working hours per week, majority almost a full working week. Looking at eductaion and age, this can be explained by ongoing studies, or parent leave. Id **10845** put 35 hours per week, albeit unemployed, not in family and not ongoing education. One possible explanation of this the divorced marital status, if the individual has custody over his children.

# + tags=[]
# pd.qcut(df['hours-per-week'], q=4)
df['hours-per-week'].describe()
# -

try:
    df['occupation'] = df['occupation'].cat.add_categories(['Not-stated'])
    df['occupation'] = df['occupation'].fillna('Not-stated')
except ValueError as e:
    pass
df['occupation'].value_counts().plot.barh(title='occupation', figsize=(5, 5)) # bar chart for exploration

try:
    df['workclass'] = df['workclass'].cat.add_categories(['Not-stated'])
    df['workclass'] = df['workclass'].fillna('Not-stated')
except ValueError as e:
    pass
display(df['workclass'].fillna('Not-stated').value_counts().plot.barh(title='work class', figsize=(5, 5)))

# ## Market Basket Analysis
# Exploring factor combinations linked to > 50K year income.

display(df['age'].describe())
display(df['age'].plot.hist())


# +
def age_categorize(age: int) -> str:
    if age < 25:
        return '-25'
    if 25 <= age <= 30:
        return '25-30'
    if 31 <= age <= 40:
        return '31-40'
    if 41 <= age <= 50:
        return '41-50'
    if 51 <= age <= 60:
        return '51-60'
    if 61 <= age:
        return '61+'

df['age-category'] = df['age'].apply(age_categorize)
df['age-category'] = df['age-category'].astype('category')
# -

# Native country feature will not be selected. The visualization below suggests that natives other than US will not get sufficient support.

df['native-country'].value_counts().plot.bar()

basket_df = pd.get_dummies(df[['age-category', 'marital-status', 'race', 'sex', 'income >50K', 'occupation', 'workclass']])
basket_df = basket_df.rename(columns={'income >50K_0': 'income <50K', 'income >50K_1': 'income >50K'})
basket_df = basket_df.drop(['workclass_Not-stated', 'occupation_Not-stated'], axis=1)
display(basket_df.isnull().values.any())
display(basket_df.head())

basket_df['capital-gain'] = df['capital-gain'].apply(lambda x: 1 if x > 0 else 0)
basket_df['capital-loss'] = df['capital-loss'].apply(lambda x: 1 if x > 0 else 0)

df['education'].cat.categories

education_dict_ordered

basic_edu = [v for k, v in education_dict_ordered.items() if k < 10]
higher_edu = [v for k, v in education_dict_ordered.items() if k >= 10 and k < 14]
university_edu = [v for k, v in education_dict_ordered.items() if k >= 14]
university_edu

basket_df['education_Basic'] = df['education'].apply(lambda x: 1 if x in basic_edu else 0)
basket_df['education_Higher'] = df['education'].apply(lambda x: 1 if x in higher_edu else 0)
basket_df['education_University'] = df['education'].apply(lambda x: 1 if x in university_edu else 0)
basket_df[['education_Basic', 'education_Higher', 'education_University']].head()

# + tags=[]
basket_df.head()


# + tags=[]
def union(df, *itemset) -> pd.DataFrame:
    tmp_df = pd.DataFrame()
    tmp_df['conj'] = df[itemset[0]]
    for item in itemset:
        tmp_df['conj'] *= df[item]
    return tmp_df['conj']

union(basket_df, 'age-category_-25', 'sex_Male')


# +
def support(df, *itemset) -> float:
    """
    frequency of conjunction of given items
    
    features: list(str)
    df: pandas.DataFrame
    return: float
    """
    return sum(union(df, *itemset))/len(df.index)

# test
support(basket_df, 'age-category_-25') == basket_df['age-category_-25'].sum()/basket_df['age-category_-25'].count()


# +
def confidence(df, X: List[str], Y: List[str]) -> float:
    """
    likelihood of Y given X
    X: itemset
    Y: itemset
    """
    return support(df, *[*X, *Y])/support(df, *X)

confidence(basket_df, ['age-category_-25'], ['income >50K'])


# +
def lift(df, X: List[str], Y: List[str]) -> float:
    """
    likelihood of Y given X, taking into account the frequency of Y
    """
    return support(df, *[*X, *Y])/(support(df, *X) * support(df, *Y))
    

lift(basket_df, ['race_White'], ['income >50K'])

# +
supp_threshold = 0.05

# steap 1-2
frequencies = basket_df.sum()/len(basket_df.index) # support table
frequencies = frequencies[frequencies > supp_threshold] # filter threshold
frequencies
# -

# step 3 create combination pairs
itemset = [*itertools.combinations(frequencies.index, 2)]
itemset[0:5]

freqs = pd.Series(data= [support(basket_df, *x) for x in itemset], index=itemset)
freqs


def apriori_rules(df: pd.DataFrame, threshold: int, ceiling=math.inf):
    r = 2
    frequencies = pd.Series(data=[support(df, x) for x in df.columns], index=df.columns)
    frequencies = frequencies[frequencies > threshold] # filter threshold
    items = {x for x in frequencies.index}
    itemset = {frozenset(x) for x in itertools.combinations(items, r)}
    rules = pd.Series(data= [support(df, *x) for x in itemset], index=itemset, dtype='float64')
    rules = rules[rules > threshold]
    r += 1
    check_subsets = set()
    while r < ceiling:
        print(r, end='\r')
        itemset = {frozenset(x) for x in itertools.combinations(items, r) if not any([a.issubset(frozenset(x)) for a in check_subsets])}
        new_rules = pd.Series(dtype='float64')
        check_subsets = set()
        items = set()
        for x in itemset:
            supp = support(df, *x)
            if supp > threshold:
                items = items.union(x)
                new_rules = new_rules.append(pd.Series([supp], index=[x]))
            else:
                check_subsets.add(x)      
        if len(new_rules) == 0:
            return rules
        rules = rules.append(new_rules)
        r += 1
    print(end='\n')
    return rules
    


rules = apriori_rules(basket_df, 0.05)
len(rules)

income_indices_over50k = [x for x in rules.index if 'income >50K' in x]
income_indices_under50k = [x for x in rules.index if 'income <50K' in x]
len(income_indices_over50k), len(income_indices_under50k)

# +
income_indices_over50k[0].difference(['income >50K'])

lift_over50k = pd.DataFrame(data = { 'lift': [lift(basket_df, x.difference(['income >50K']), ['income >50K']) for x in income_indices_over50k], 'confidence': [confidence(basket_df, x.difference(['income >50K']), ['income >50K']) for x in income_indices_over50k], 'support': [support(basket_df, *x) for x in income_indices_over50k]}, index=income_indices_over50k)
lift_under50k = pd.DataFrame(data = { 'lift': [lift(basket_df, x.difference(['income <50K']), ['income <50K']) for x in income_indices_under50k], 'confidence':  [confidence(basket_df, x.difference(['income <50K']), ['income <50K']) for x in income_indices_under50k], 'support': [support(basket_df, *x) for x in income_indices_under50k]}, index=income_indices_under50k)
# -

# ## Results and interpretation

lift_over50k.sort_values(['lift', 'confidence', 'support'], ascending=[False, False, False]).head(5)

lift_under50k.sort_values(['lift', 'confidence', 'support'], ascending=[False, False, False]).head(5)

lift_over50k.to_csv('data/mba_over50k.csv')
lift_under50k.to_csv('data/mba_under50k.csv')

# The tables show that all identified relationship suffer from shortcomings. The high lifts for income over 50K may indicate high strengths of those relationships, however, the confidence is low.
# Even though the confidences in the under 50K table are near 1, the lifts around 1.3 point to arguably weak links.



StarcoderdataJupyterScriptsDedupFiltered