<jupyter_start><jupyter_text>Importing Libraries<jupyter_code>import pandas as pd
import numpy as np
import sklearn

# Visualization Libraies
import matplotlib.pyplot as plt
import seaborn as sns
from pandas.plotting import scatter_matrix

# to split the dataset into test and train 
from sklearn.model_selection import train_test_split

# For Custom Transformer
from sklearn.base import BaseEstimator, TransformerMixin

# Transformers for the Pipeline
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import StandardScaler
import category_encoders
from category_encoders import OrdinalEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer

# Evaluation metrics
from sklearn.metrics import accuracy_score

# Cross Validation libraries
from sklearn.model_selection import cross_val_score

# Predictors 
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.linear_model import LogisticRegression

# For deployment 
import joblib           # pickle the pipeline
from joblib import dump<jupyter_output><empty_output><jupyter_text>Loading in the Dataset source: https://www.kaggle.com/kemical/kickstarter-projects/notebooks?sortBy=dateRun&group=upvoted&pageSize=20&datasetId=4104<jupyter_code># To view all the columns 
pd.set_option('display.max_columns', None)

df = pd.read_csv('kickstarter.csv', 
                parse_dates=['deadline', 'launched'])
df.head(5)<jupyter_output><empty_output><jupyter_text>Ideas:- delete ID- feature engineering (deadline and launched)- Consider top five countries- Only consider failed or successful and make it a binary classifictaion- Will keep all the 15 main categories- For category we could pass it to the ordinal encoding   EDA<jupyter_code>total_nan = df.isna().sum().sort_values(ascending=False)
percentage_nan = (total_nan / df.shape[0]) * 100
tabel = pd.concat([total_nan, percentage_nan], axis=1, keys=['Total NAN', 'Percentage of NAN'])
tabel
df.state.value_counts()
# filtering out the dataset for binary target variable - failed / successful
df = df.loc[(df['state'] == 'failed') | (df['state'] == 'successful')]
df.state.value_counts()
# Imbalance classification but not worrisome
successful = 131490/(192871+131490)
successful
# changing the target variable to 0 and 1 
df['state'] = df['state'].map({'failed':0, 'successful':1})
df['state'].value_counts()<jupyter_output><empty_output><jupyter_text>Checking for leakage<jupyter_code>df.loc[df['usd_pledged_real'] < df['usd_goal_real']]['state'].value_counts()
# checking for backers
df.loc[df['backers'] > 70]['state'].value_counts()
df.loc[df['usd_pledged_real'] > 25000]['state'].value_counts()
df.loc[df['usd_pledged_real'] > 25000].shape<jupyter_output><empty_output><jupyter_text>There is clearly a data leakage here.   Deleting verticals that we don't need - Dropping USD pledged and pledged as usd_pledged_real has the same information- Dropping country as currency and country are highly correlated- Dropping goal as we have goal converted in USD as a vertical- Dropping backers and<jupyter_code>df = df.drop(['usd pledged', 'pledged', 'country', 'goal', 'ID', 'usd_pledged_real', 'backers'], axis=1) 
df.shape
# dropping 4 nan values in name
df.dropna(axis=0, inplace=True)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 331672 entries, 0 to 378660
Data columns (total 8 columns):
 #   Column         Non-Null Count   Dtype         
---  ------         --------------   -----         
 0   name           331672 non-null  object        
 1   category       331672 non-null  object        
 2   main_category  331672 non-null  object        
 3   currency       331672 non-null  object        
 4   deadline       331672 non-null  datetime64[ns]
 5   launched       331672 non-null  datetime64[ns]
 6   state          331672 non-null  int64         
 7   usd_goal_real  331672 non-null  float64       
dtypes: datetime64[ns](2), float64(1), int64(1), object(4)
memory usage: 22.8+ MB<jupyter_text>Checking high cardinality<jupyter_code>cols = ['category', 'main_category', 'currency']

for col in cols:
    print(f"{col} has {df[col].nunique()} unique values")
# Either we can delete category vertical or ordinal encode it!
df.category.value_counts()
# Lets consider top 5 currencies and delete the rest 
df.currency.value_counts()
# Filtering out top 5 currencies 
df = df.loc[(df['currency'] == 'USD') | (df['currency'] == 'GBP') | (df['currency'] == 'EUR') \
      | (df['currency'] == 'CAD') | (df['currency'] == 'AUD')]
# changing the currency names into countries so it will be easier to ask a user for input
df['currency'] = df['currency'].map({
                'USD':'USA',
                'GBP':'UK',
                'EUR':'Europe',
                'CAD':'Canada',
                'AUD': 'Australia'})
df.currency.value_counts()
# reset index as we deleted a few rows when we filtered
df.reset_index(drop=True, inplace=True)<jupyter_output><empty_output><jupyter_text>Feature Engineering - calculating the length of the campaign from deadline and launched - calculating the total number of words in name<jupyter_code># making a new column length_days
df['length_days'] = (df['deadline'] - df['launched']).dt.days + 1

# deleting deadline and launched cols
df = df.drop(['deadline', 'launched'], axis=1)
# Total number of words in each row in name
df['name'] = df['name'].str.split().str.len()<jupyter_output><empty_output><jupyter_text>Baseline Model<jupyter_code># we need our model to beat this score
baseline = 1 - successful
baseline<jupyter_output><empty_output><jupyter_text>Split Dataset<jupyter_code># Feature Matrix and Target Variable
X = df.drop('state', axis=1)
y = df['state']
# Splitting the dataset into train and test dataset 

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=105)
print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)
X_train.head()<jupyter_output><empty_output><jupyter_text>Pipeline<jupyter_code># looking at numerical attributes for simple imputer with median 
num_attribs = X_train.select_dtypes(exclude='object')
num_attribs.columns
# looking at categorical attributes for simple imputer with 'most_frequent'
cat_attribs = X_train.select_dtypes(include='object')
cat_attribs.columns
# making seperate list of cols for ohe and ordinal encoding
cat_attribs_ohe = X_train[['main_category', 'currency']]
cat_attribs_ohe.columns
cat_attribs_ord = X_train[['category']]
cat_attribs_ord.columns
"""Building a custom Transformer that will give the output in a dataframe after 
applying the simple imputer so we could pass it to the categorical_encoders 
which does not accept np.array""" 

class ImputerDF(BaseEstimator, TransformerMixin):
    def __init__(self):
        self.imputer = SimpleImputer(strategy='most_frequent')
        self.cols = []
        
    def fit(self, X, y=None):
        self.imputer.fit(X)
        self.cols = list(X.columns)
        return self
    
    def transform(self, X):
        X_t = self.imputer.transform(X)
        return pd.DataFrame(X_t, columns=self.cols)
# Using median as the strategy for Simple Imputer to predict NaN values
num_pipeline = make_pipeline(
                            SimpleImputer(strategy='median'),
                            StandardScaler()
)
""" Adding the Custom Transformer to impute using 'most_frequent' strategy
 and giving out an output as a dataframe instead of an array """
cat_pipeline = make_pipeline(
                            ImputerDF(),
                            OrdinalEncoder(cols = cat_attribs_ord),
                            OneHotEncoder(cols = cat_attribs_ohe)
)
# generating a list of categorical and numerical columns to pass it in the column transformer
cat_attributes = list(cat_attribs)
num_attributes = list(num_attribs)

# putting two pipelines together using ColumnTransformer
final_pipeline = ColumnTransformer([
                            ('num_pipeline', num_pipeline, num_attributes),
                            ('cat_pipeline', cat_pipeline, cat_attributes)
])
# Fit and trasnform on X_train
X_train_transformed = final_pipeline.fit_transform(X_train)

print(X_train_transformed.shape)<jupyter_output>(259486, 24)<jupyter_text>Model Selection- Ran a few simple models and picked the most generalized model- Using **Accuracy** as the evaluation metrics as the dataset is balanced.  Random Forest Classifier<jupyter_code>forest_clf = RandomForestClassifier(n_jobs=-1, 
                                    random_state=105, 
                                    n_estimators=80, 
                                    max_depth=11)

forest_clf.fit(X_train_transformed,y_train)

y_pred_train = forest_clf.predict(X_train_transformed)

score = accuracy_score(y_train, y_pred_train)
score_forest = cross_val_score(forest_clf,
                         X_train_transformed,
                         y_train,
                         scoring='accuracy',
                         cv=10,
                         n_jobs=-1)
print(f"""
    Accuracy score on training set: {score}\n\n
    Accuracy score on validation set: {score_forest.mean()}
    """)<jupyter_output>Accuracy score on training set: 0.6768110803665708


    Accuracy score on validation set: 0.6694195546845952<jupyter_text>Logistic Regression<jupyter_code>lr = LogisticRegression(max_iter=10000)

lr.fit(X_train_transformed, y_train)

y_pred_train = lr.predict(X_train_transformed)

score = accuracy_score(y_train, y_pred_train)
score_lr = cross_val_score(lr,
                         X_train_transformed,
                         y_train,
                         scoring='accuracy',
                         cv=10,
                         n_jobs=-1)
print(f"""
    Accuracy score on training set: {score}\n\n
    Accuracy score on validation set: {score_lr.mean()}
    """)<jupyter_output>Accuracy score on training set: 0.6494647110056034


    Accuracy score on validation set: 0.6493375460133903<jupyter_text>Gradient Boost Classifier<jupyter_code>gb = GradientBoostingClassifier()

gb.fit(X_train_transformed, y_train)

y_pred_train = gb.predict(X_train_transformed)

score = accuracy_score(y_train, y_pred_train)
score_gb = cross_val_score(gb,
                         X_train_transformed,
                         y_train,
                         scoring='accuracy',
                         cv=10)
print(f"""
    Accuracy score on training set: {score}\n\n
    Accuracy score on validation set: {score_gb.mean()}
    """)<jupyter_output>Accuracy score on training set: 0.6755085052758145


    Accuracy score on validation set: 0.6754044641751807<jupyter_text>SGD Classifier<jupyter_code>sgd = SGDClassifier(n_jobs=-1)

sgd.fit(X_train_transformed, y_train)

y_pred_train = sgd.predict(X_train_transformed)

score = accuracy_score(y_train, y_pred_train)
score_sgd = cross_val_score(sgd,
                         X_train_transformed,
                         y_train,
                         scoring='accuracy',
                         cv=10)
print(f"""
    Accuracy score on training set: {score}\n\n
    Accuracy score on validation set: {score_sgd.mean()}
    """)<jupyter_output>Accuracy score on training set: 0.6419267320780312


    Accuracy score on validation set: 0.6177520792748281<jupyter_text>Voting Classifiers using soft voting.<jupyter_code>lr = LogisticRegression(max_iter=10000)

forest_clf = RandomForestClassifier(n_jobs=-1, 
                                    random_state=105, 
                                    n_estimators=60, 
                                    max_depth=10)

sgd = SGDClassifier(loss="modified_huber", 
                   n_jobs=-1)

gb = GradientBoostingClassifier()

voting_clf = VotingClassifier(
                estimators=[("lr",lr), ("rf",forest_clf), 
                            ("sgd",sgd), ("gb", gb)],
                voting='soft',
                n_jobs=-1)

voting_clf.fit(X_train_transformed, y_train)

y_pred_train = voting_clf.predict(X_train_transformed)

score = accuracy_score(y_train, y_pred_train)
score_voting_soft = cross_val_score(voting_clf,
                         X_train_transformed,
                         y_train,
                         scoring='accuracy',
                         cv=10,
                        n_jobs=-1)
print(f"""
    Accuracy score on training set: {score}\n\n
    Accuracy score on validation set: {score_voting_soft.mean()}
    """)<jupyter_output>Accuracy score on training set: 0.6534649268168611


    Accuracy score on validation set: 0.6525014962618656<jupyter_text>Voting Classifier using hard voting.<jupyter_code>lr = LogisticRegression(max_iter=10000)

forest_clf = RandomForestClassifier(n_jobs=-1, 
                                    random_state=105,
                                    max_depth=15)

sgd = SGDClassifier(n_jobs=-1) 

gb = GradientBoostingClassifier()

voting_clf_hard = VotingClassifier(
                estimators=[("lr",lr), ("rf",forest_clf), 
                            ("sgd",sgd), ("gb", gb)],
                voting='hard',
                n_jobs=-1)

voting_clf_hard.fit(X_train_transformed, y_train)

y_pred_train = voting_clf_hard.predict(X_train_transformed)

score = accuracy_score(y_train, y_pred_train)
score_voting_hard = cross_val_score(voting_clf_hard,
                         X_train_transformed,
                         y_train,
                         scoring='accuracy',
                         cv=10,
                        n_jobs=-1)
print(f"""
    Accuracy score on training set: {score}\n\n
    Accuracy score on validation set: {score_voting_hard.mean()}
    """)<jupyter_output>Accuracy score on training set: 0.6660166637121078


    Accuracy score on validation set: 0.6634885523246815<jupyter_text>Selecting the most generalized model<jupyter_code>fig = plt.figure(figsize=(10, 4), facecolor='#dadada')

plt.plot([1] * 10, score_forest, ".")
plt.plot([2] * 10, score_lr, ".")
plt.plot([3] * 10, score_gb, ".")
plt.plot([4] * 10, score_sgd, ".")
plt.plot([5] * 10, score_voting_soft, ".")
plt.plot([6] * 10, score_voting_hard, ".")

plt.boxplot(
    [score_forest, score_lr, score_gb, score_sgd,
     score_voting_soft, score_voting_hard],
    labels=("Random Forest", "Logistic Reg", 'Grad Boost',
            'Stoch Grad Descent', 'Soft Voting Clf', 'Hard Voting Clf'))

plt.ylabel("Accuracy Score", fontsize=14)
plt.title('Algorithm Comparison')

# Remove the splines
plt.gca().spines["top"].set_visible(False)
plt.gca().spines["bottom"].set_visible(False)
plt.gca().spines["right"].set_visible(False)
plt.gca().spines["left"].set_visible(False)

plt.tight_layout()  # Makes it better looking on laptops

plt.show()<jupyter_output><empty_output><jupyter_text>Evaluating our model on Test set I picked the gradient boost classifer as it is the most generalized model.<jupyter_code># transforming X_test 
X_test_transformed = final_pipeline.transform(X_test)

y_test_pred =gb.predict(X_test_transformed)

score = accuracy_score(y_test, y_test_pred)
print(f"""
    Accuracy score on the TRAINING set using gb Classifier: {score_gb.mean()}\n\n
    Accuracy score on TEST set using gb Classifier: {score}
    """)<jupyter_output>Accuracy score on the TRAINING set using gb Classifier: 0.6754044641751807


    Accuracy score on TEST set using gb Classifier: 0.6757306696263411<jupyter_text>Final Pipeline for Deployement<jupyter_code>deploy_pipeline = make_pipeline(
                        final_pipeline, 
                        gb)  # Final Pick 

# Fitting the Pipeline 
deploy_pipeline.fit(X_train, y_train)
# Pickle the Pipeline using joblib
dump(deploy_pipeline, 'pipeline.joblib', compress=True)<jupyter_output><empty_output><jupyter_text>Get version of the packages used in the pipeline so we could then install all those packages in virtual env<jupyter_code>print(f'joblib=={joblib.__version__}')
print(f'scikit-learn=={sklearn.__version__}')
print(f'category_encoders=={category_encoders.__version__}')<jupyter_output>joblib==0.16.0
scikit-learn==0.23.1
category_encoders==2.2.2
StarcoderdataJupyterStructuredCleanDedup