# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import warnings
warnings.simplefilter('ignore')

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.cluster import DBSCAN
from sklearn.ensemble import IsolationForest

# %matplotlib inline
# -

# # Load Dataset

info = pd.read_csv('./10.CreditCard/cc_info.csv')
info.head()

info.info()

info.describe()

transaction = pd.read_csv('./10.CreditCard/transactions.csv', parse_dates=['date'])
transaction.head()

transaction.info()

transaction.describe()

# # Data Processing

data = pd.merge(left=transaction, right=info, on='credit_card', how='left')
data.head()

# +
# extract month, weekday, and hour information
data['month'] = data['date'].apply(lambda x: x.month)
data['day'] = data['date'].apply(lambda x: x.day)
data['weekday'] = data['date'].apply(lambda x: x.dayofweek)
data['hour'] = data['date'].apply(lambda x: x.hour)

# drop useless features
data = data.drop(labels='date', axis=1)

data.head()


# -

# # Question 1
# Your boss wants to identify those users that in your dataset never went above the monthly credit card limit (calendar month). The goal of this is to automatically increase their limit. Can you send him the list of Ids?

def monthly_transaction(df):
    """ function to calculate montly total transactions """
    total_trans = df['transaction_dollar_amount'].sum()
    credit_limit = df['credit_card_limit'].values[0]
    
    return pd.Series([total_trans, credit_limit], index=['total_transaction', 'credit_limit'])


# +
# find users who never exceed credit limit
grouped = data.groupby(['credit_card', 'month']).apply(monthly_transaction)
grouped = grouped.reset_index()

grouped['exceed'] = (grouped['total_transaction'] > grouped['credit_limit']).astype(int)
grouped_sum = grouped.groupby('credit_card')['exceed'].sum().reset_index()

lists = grouped_sum[grouped_sum['exceed'] == 0][['credit_card']]

print('Valid users:\t', len(lists))
lists.head()


# -

# So, there are 862 credit cards in total that never exceed their credit limit.

# # Question 2
# On the other hand, she wants you to implement an algorithm that as soon as a user goes above her monthly limit, it triggers an alert so that the user can be notified about that. We assume here that at the beginning of the new month, user total money spent gets reset to zero (i.e. she pays the card fully at the end of each month). Build a function that for each day, returns a list of users who went above their credit card monthly limit on that day.

def alert(data, month, day):
    """ function to return the list of credit cards that exceed the limit """
    data = data[(data['month'] == month) & (data['day'] <= day)]
    limit = data.groupby('credit_card')['credit_card_limit'].min().reset_index()
    transaction = data.groupby('credit_card')['transaction_dollar_amount'].sum().reset_index()
    
    merged = pd.merge(left=transaction, right=limit, on='credit_card', how='left')
    result = merged[merged['transaction_dollar_amount'] > merged['credit_card_limit']]
    
    return result


# example
alert(data, month=8, day=20)

# # Question 3
# Finally, your boss is very concerned about frauds cause they are a huge cost for credit card companies. She wants you to implement an unsupervised algorithm that returns all transactions that seem unusual and are worth being investigated further.

data.head()

# There are some features that cannot be directly used, such as `Long`, `Lat`, `city`, `state`, `zipcode`. However, these information is important towards anomaly detection. One idea is to replace `city`, `state`, `zipcode` with the user's home latitude and longitude. In this work, to simplify the work, I use the median of `Long` and `Lat` as the indicator of user's home address.

# calculate the median for Longitude and Latitude
address = data.groupby('credit_card')['Long', 'Lat'].mean().reset_index()
address = address.rename(columns={'Long': 'Long_median', 'Lat': 'Lat_median'})
data = pd.merge(left=data, right=address, on='credit_card', how='left')

# calculate the distance
data['distance'] = np.sqrt((data['Long'] - data['Long_median'])**2 + (data['Lat'] - data['Lat_median'])**2)

# For transaction and credit limit information, two more useful information are extracted:
# * ratio of transaction and transaction median
# * ratio of transaction and credit limit

# calculate user transaction median
median = data.groupby('credit_card')['transaction_dollar_amount'].median().reset_index()
median = median.rename(columns={'transaction_dollar_amount': 'transaction_median'})
data = pd.merge(left=data, right=median, on='credit_card', how='left')

# +
# extract useful information
data['transaction_to_median'] = data['transaction_dollar_amount'] / data['transaction_median']
data['transaction_to_limit'] = data['transaction_dollar_amount'] / data['credit_card_limit']

data.head()
# -

# drop useless features
labels = ['transaction_dollar_amount', 'Long', 'Lat', 'city', 'state', 'zipcode', 'credit_card_limit', 
          'month', 'day', 'Long_median', 'Lat_median', 'transaction_median']
data = data.drop(labels=labels, axis=1)
data.head()

# Now, let's first visualize the data in 2D throught PCA.

# normalize the data
x = data[['weekday', 'hour', 'distance', 'transaction_to_median', 'transaction_to_limit']]
# x = data[['distance', 'transaction_to_median', 'transaction_to_limit']]
scaler = StandardScaler()
norm_x = scaler.fit_transform(x)

# +
# apply PCA
pca = PCA(n_components=2, random_state=42)
pca_x = pca.fit_transform(norm_x)

fig, ax = plt.subplots(figsize=(10, 8))
ax.plot(pca_x[:, 0], pca_x[:, 1], '.', markersize=1)
ax.set_xlabel('PCA Component 1', fontsize=12)
ax.set_ylabel('PCA Component 2', fontsize=12)
plt.show()
# -

# There are different methods for unsupervised anomaly detection, such as Isolation Forest, One-Class SVM, DBSCAN and so on.
#
# Here, for simplicity, only DBSCAN method with default parameters is implemented.

# build isolation forest model
forest = IsolationForest(n_estimators=100, max_samples='auto', contamination=0.1, 
                         max_features=1.0, bootstrap=False, n_jobs=-1, random_state=42)
forest = forest.fit(norm_x)
score = forest.decision_function(norm_x)
label = forest.predict(norm_x)

# +
# visualization
inlier = pca_x[label == 1]
outlier = pca_x[label == -1]
xmin = pca_x[:, 0].min() - 1
xmax = pca_x[:, 0].max() + 1
ymin = pca_x[:, 1].min() - 1
ymax = pca_x[:, 1].max() + 1

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(18, 7))
ax[0].plot(inlier[:, 0], inlier[:, 1], 'b.', markersize=2, label='Inlier')
ax[0].set_xlabel('PCA Component 1', fontsize=14)
ax[0].set_ylabel('PCA Component 2', fontsize=14)
ax[0].legend(fontsize=14)
ax[0].set_xlim([xmin, xmax])
ax[0].set_ylim([ymin, ymax])
ax[1].plot(outlier[:, 0], outlier[:, 1], 'r.', markersize=2, label='Outlier')
ax[1].set_xlabel('PCA Component 1', fontsize=14)
ax[1].set_ylabel('PCA Component 2', fontsize=14)
ax[1].legend(fontsize=14)
ax[1].set_xlim([xmin, xmax])
ax[1].set_ylim([ymin, ymax])
plt.tight_layout()
plt.show()
# -

# For other methods, please refer to: [Link](https://github.com/stasi009/TakeHomeDataChallenges/blob/master/10.CreditCard/credit_card_fraud.ipynb)



StarcoderdataJupyterScriptsDedupFiltered