# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# + [markdown] deletable=true editable=true
# Machine Learning Challenge #2 from HackerEarth
#
# Data: Kickstarter Project Details
#
# Target: Project will successfully get funded or not
#
# Hardware Configuration: 8-core CPU, 16gb RAM
#
# https://www.hackerearth.com/challenge/competitive/machine-learning-challenge-2/problems/

# + deletable=true editable=true
import numpy as np
import pandas as pd
from matplotlib import pyplot

# + [markdown] deletable=true editable=true
# ## 1. Data Exploration

# + deletable=true editable=true
train = pd.read_csv('../../input/train.csv')
test = pd.read_csv('../../input/test.csv')

# + deletable=true editable=true
print(train.shape, test.shape)

# + deletable=true editable=true



# + deletable=true editable=true
train[:2]

# + [markdown] deletable=true editable=true
# ## 2. Data Preprocessing
# LabelEncoder and OneHotEncoder

# + deletable=true editable=true
from sklearn.preprocessing import LabelEncoder, OneHotEncoder

# + deletable=true editable=true
#merge train and test data for preprocessing
size_train = train.shape[0]
target = train.final_status.values
train.drop(['backers_count', 'final_status'], axis=1, inplace=True)
data = train.append(test, ignore_index=True)

# + deletable=true editable=true
#project_id - to int
data['project_id'] = data['project_id'].apply(lambda x: x[4:]).apply(int)

# + deletable=true editable=true
#disable_communication - label encode
le = {}
le['disable_communication'] = LabelEncoder()
data['disable_communication'] = le['disable_communication'].fit_transform(data['disable_communication'])

# + deletable=true editable=true
#country, currency - label & one hot encode
ohe = {}
for col in ['country', 'currency']:    
    #fit encoder
    le[col] = LabelEncoder()
    ohe[col] = OneHotEncoder(sparse=False)
    
    #process train data
    data[col] = le[col].fit_transform(data[col])
    features = ohe[col].fit_transform(data[col].values.reshape(-1,1))
    print(data.shape, ' + ', features.shape)
    data = pd.concat([data, pd.DataFrame(features, columns=[col+'_'+str(n) for n in range(features.shape[1])])], axis=1)
print(data.shape)

# + [markdown] deletable=true editable=true
# ## 3. Feature Extraction

# + [markdown] deletable=true editable=true
# ### 3.1 Date Features

# + deletable=true editable=true
from datetime import datetime
for col in ['deadline', 'state_changed_at', 'created_at', 'launched_at']:
    dt = data[col].apply(datetime.fromtimestamp)
    data[col+'_year'] = dt.apply(lambda x: x.year)
    data[col+'_month'] = dt.apply(lambda x: x.month)
    data[col+'_day'] = dt.apply(lambda x: x.day)
    data[col+'_hour'] = dt.apply(lambda x: x.hour)
    data[col+'_minute'] = dt.apply(lambda x: x.minute)
    data[col+'_second'] = dt.apply(lambda x: x.second)

# + deletable=true editable=true
data['ready_duration'] = data['launched_at'] - data['created_at']

# + deletable=true editable=true
data['run_duration'] = data['state_changed_at'] - data['launched_at']

# + deletable=true editable=true
data['goal_duration'] = data['deadline'] - data['launched_at']

# + deletable=true editable=true
data['goal_daily'] = np.round(data['goal'] / np.round((data['run_duration']/(60*60*24)), decimals=1), decimals=0).astype(int)

# + [markdown] deletable=true editable=true
# ### 3.2 Text Features

# + deletable=true editable=true
from sklearn.feature_extraction.text import CountVectorizer

# + deletable=true editable=true
period = data['launched_at_year'].apply(str) + data['launched_at_month'].apply(lambda x: str(x).zfill(2))
period_vectorizer = CountVectorizer()
period_result = period_vectorizer.fit_transform(period)

period_value = {}
for value in period.unique():
    period_value[value] = (period == value).sum()
    
data['launched_at_ym_same'] = period.apply(lambda x: period_value[x])

# + deletable=true editable=true
period = data['deadline_year'].apply(str) + data['deadline_month'].apply(lambda x: str(x).zfill(2))
period_vectorizer = CountVectorizer()
period_result = period_vectorizer.fit_transform(period)

period_value = {}
for value in period.unique():
    period_value[value] = (period == value).sum()
    
data['deadline_ym_same'] = period.apply(lambda x: period_value[x])

# + deletable=true editable=true
#text features
data['keywords'] = data['keywords'].apply(str).apply(lambda x: x.split('-'))

# + deletable=true editable=true
for col in ['name', 'desc', 'keywords']:
    data[col+"_len"] = data[col].apply(str).apply(len)
    data[col+"_count"] = data[col].apply(str).apply(lambda x: len(x.split(' ')))

# + [markdown] deletable=true editable=true
# ### 3.3 Dimentionality Reduction Features

# + deletable=true editable=true
from sklearn.decomposition import PCA, FastICA, TruncatedSVD
from sklearn.random_projection import GaussianRandomProjection, SparseRandomProjection

# + deletable=true editable=true
n_comp = 30

# tSVD
tsvd = TruncatedSVD(n_components=n_comp, random_state=420)
tsvd_results = tsvd.fit_transform(data.drop(['name', 'desc', 'keywords'], axis=1))

# PCA
pca = PCA(n_components=n_comp, random_state=420)
pca_results = pca.fit_transform(data.drop(['name', 'desc', 'keywords'], axis=1))

# ICA
ica = FastICA(n_components=n_comp, random_state=420)
ica_results = ica.fit_transform(data.drop(['name', 'desc', 'keywords'], axis=1))

# GRP
grp = GaussianRandomProjection(n_components=n_comp, eps=0.1, random_state=420)
grp_results = grp.fit_transform(data.drop(['name', 'desc', 'keywords'], axis=1))

# SRP
srp = SparseRandomProjection(n_components=n_comp, dense_output=True, random_state=420)
srp_results = srp.fit_transform(data.drop(["name", 'desc', 'keywords'], axis=1))

# Append decomposition components to datasets
for i in range(n_comp):
    data['pca_' + str(i)] = pca_results[:, i]
    data['ica_' + str(i)] = ica_results[:, i]
    data['tsvd_' + str(i)] = tsvd_results[:, i]
    data['grp_' + str(i)] = grp_results[:, i]
    data['srp_' + str(i)] = srp_results[:, i]

# + [markdown] deletable=true editable=true
# ### 3.4 Bag of words

# + deletable=true editable=true
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation, SparsePCA
from sklearn.preprocessing import Normalizer
from sklearn.cluster import KMeans

# + [markdown] deletable=true editable=true
# #### 3.4.1 Term Vectorizer

# + deletable=true editable=true
vectorizer = {}

# + deletable=true editable=true
keywords = data['keywords'].apply(lambda x: ' '.join(x))
vectorizer['keywords'] = TfidfVectorizer(stop_words='english')#, ngram_range=(1,3))
keywords_result = vectorizer['keywords'].fit_transform(keywords)

# + deletable=true editable=true
vectorizer['desc'] = TfidfVectorizer(stop_words='english')#, ngram_range=(1,3))
desc_result = vectorizer['desc'].fit_transform(data['desc'].fillna(''))

# + deletable=true editable=true
print(keywords_result.shape, desc_result.shape)


# + [markdown] deletable=true editable=true
# #### 3.4.2 Topic Extraction

# + deletable=true editable=true
def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        print(" ".join([feature_names[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
    print()


# + [markdown] deletable=true editable=true
# ##### 3.4.2.1 KMeans Clustering

# + deletable=true editable=true
# Fit KMeans Cluster model - keywords, desc
cluster_svd = {
    'keywords': TruncatedSVD(n_components=200),
    'desc': TruncatedSVD(n_components=200) #2gb ram
}
cluster_svd_result = {
    'keywords': cluster_svd['keywords'].fit_transform(keywords_result),
    'desc': cluster_svd['desc'].fit_transform(desc_result)
}

# + deletable=true editable=true
cluster_norm = {
    'keywords': Normalizer(copy=False),
    'desc': Normalizer(copy=False)
}
cluster_norm_result = {
    'keywords': cluster_norm['keywords'].fit_transform(cluster_svd_result['keywords']),
    'desc': cluster_norm['desc'].fit_transform(cluster_svd_result['desc'])
}
cluster = {
    'keywords': KMeans(n_clusters=40, init='k-means++', max_iter=300, n_init=10,
                        verbose=1, n_jobs=-1),
    'desc': KMeans(n_clusters=40, init='k-means++', max_iter=300, n_init=10,
                    verbose=1, n_jobs=-1)
}

# + deletable=true editable=true
cluster_result = {}
cluster_result['keywords'] = cluster['keywords'].fit_transform(cluster_norm_result['keywords'])

# + deletable=true editable=true
cluster_result['desc'] = cluster['desc'].fit_transform(cluster_norm_result['desc'])

# + deletable=true editable=true
print("Top terms per cluster:")

c = 'desc' #'keywords

original_space_centroids = cluster_svd[c].inverse_transform(cluster[c].cluster_centers_)
order_centroids = original_space_centroids.argsort()[:, ::-1]

terms = vectorizer[c].get_feature_names()
for i in range(40):
    print("Cluster %d:" % i, end='')
    for ind in order_centroids[i, :10]:
        print(' %s' % terms[ind], end='')
    print()

# + deletable=true editable=true
#append to data
data = pd.concat([data, pd.DataFrame(cluster_result['keywords'], 
                                     columns=['cluster_k_'+str(n) for n in range(cluster_result['keywords'].shape[1])])], axis=1)
data = pd.concat([data, pd.DataFrame(cluster_result['desc'], 
                                     columns=['cluster_d_'+str(n) for n in range(cluster_result['desc'].shape[1])])], axis=1)

# + [markdown] deletable=true editable=true
# ##### 3.4.2.2 NMF Decomposition

# + deletable=true editable=true
# Fit the NMF model
nmf = {}
nmf_result = {}

nmf['keywords'] = NMF(n_components=40, random_state=420,
                      alpha=.1, l1_ratio=.5, verbose=1)
nmf_result['keywords'] = nmf['keywords'].fit_transform(keywords_result)

nmf['desc'] = NMF(n_components=40, random_state=420,
                  alpha=.1, l1_ratio=.5, verbose=1)
nmf_result['desc'] = nmf['desc'].fit_transform(desc_result)

# + deletable=true editable=true
#print_top_words(nmf['keywords'], vectorizer['keywords'].get_feature_names(), 100)

# + deletable=true editable=true
#append to data
data = pd.concat([data, pd.DataFrame(nmf_result['keywords'], 
                                     columns=['nmf_k_'+str(n) for n in range(nmf_result['keywords'].shape[1])])], axis=1)
data = pd.concat([data, pd.DataFrame(nmf_result['desc'], 
                                     columns=['nmf_d_'+str(n) for n in range(nmf_result['desc'].shape[1])])], axis=1)

# + [markdown] deletable=true editable=true
# ##### 3.4.2.3 LDA Decomposition

# + deletable=true editable=true
#discarded because not well performing
# Fit the LDA model (batch_size affects speed, use more data can allocate cpu efficiently)
#lda = {}
#lda_result = {}

'''lda['keywords'] = LatentDirichletAllocation(n_topics=40, max_iter=10, max_doc_update_iter=100,
                                            learning_method='online', batch_size=keywords_result.shape[0],
                                            random_state=420, n_jobs=-1, verbose=1)
lda_result['keywords'] = lda['keywords'].fit_transform(keywords_result)'''

'''lda['desc'] = LatentDirichletAllocation(n_topics=40, max_iter=10, max_doc_update_iter=100,
                                        learning_method='online', batch_size=desc_result.shape[0],
                                        learning_offset=50.,
                                        random_state=420, n_jobs=-1, verbose=1)
lda_result['desc'] = lda['desc'].fit_transform(desc_result)'''

# + deletable=true editable=true
#print_top_words(lda['desc'], vectorizer['desc'].get_feature_names(), 100)

# + [markdown] deletable=true editable=true
# ## 4. Data Preparation - Memory Enhanced Concat

# + deletable=true editable=true
data.info()

# + deletable=true editable=true
#simple columns list
print(data.columns.values, data.columns.shape)

# + [markdown] deletable=true editable=true
# ### 4.1 Without Bag-of-Words

# + deletable=true editable=true
#split train & testdata (if skip bag of words, for feature exploration)
data_train = data[:size_train].drop(['name', 'desc', 'keywords'], axis=1).values
data_test = data[size_train:].drop(['name', 'desc', 'keywords'], axis=1).values

# + [markdown] deletable=true editable=true
# ### 4.2 With Bag-of-Words (Sparse)

# + deletable=true editable=true
#sparse matrix is GOD! use only 2gb vs > 30gb of dense array
from scipy import sparse

# + deletable=true editable=true
original = sparse.csr_matrix(data.drop(['name', 'desc', 'keywords'], axis=1).values)

# + deletable=true editable=true
concat = sparse.hstack([original, keywords_result, desc_result], format='csr')

# + deletable=true editable=true
#xgboost bug fix
data_final = sparse.hstack((concat, sparse.csr_matrix(np.ones((concat.shape[0], 1)))), format='csr')

# + deletable=true editable=true
#split train & testdata
data_train = data_final[:size_train]
data_test = data_final[size_train:]

# + deletable=true editable=true
print(data_train.shape, data_test.shape)

# + [markdown] deletable=true editable=true
# ### 4.3 Data Checkpoint

# + [markdown] deletable=true editable=true
# #### 4.3.1 To/From Without BoW

# + deletable=true editable=true
np.save('data_train_small', data_train)
np.save('data_test_small', data_test)

# + deletable=true editable=true
data_train = np.load('data_train_small')
data_test = np.load('data_test_small')

# + [markdown] deletable=true editable=true
# #### 4.3.2 To/From with Bow

# + deletable=true editable=true
#only in scipy 0.19.1
sparse.save_npz('data_train', data_train)
sparse.save_npz('data_test', data_test)

# + deletable=true editable=true
data_train = sparse.load_npz('data_train')
data_test = sparse.load_npz('data_test')

# + [markdown] deletable=true editable=true
# ## 5. Model Building

# + [markdown] deletable=true editable=true
# ### 5.1 XGBoost

# + deletable=true editable=true
import xgboost as xgb

# + [markdown] deletable=true editable=true
# #### 5.1.1 Model K-Fold Validation for initial exploration and performance checking

# + deletable=true editable=true
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split

# + deletable=true editable=true
kf = StratifiedKFold(n_splits=2, shuffle=True, random_state=420)

models_xgb = {}
i = 0
for train_index, test_index in kf.split(data_train, target):
    X_train = data_train[train_index]
    X_val = data_train[test_index]
    Y_train = target[train_index]
    Y_val = target[test_index]
    
    models_xgb[i] = xgb.XGBClassifier(max_depth=10, learning_rate=0.03, n_estimators=300,
                                      subsample=0.8, colsample_bytree=0.8, 
                                      seed=420)
    models_xgb[i].fit(X_train, Y_train, eval_metric='auc',
                  eval_set=[(X_train, Y_train), (X_val, Y_val)])
    i += 1

# + deletable=true editable=true
sorted(zip(data.columns.values, models_xgb[0].feature_importances_), key=lambda x: x[1], reverse=True)

# + [markdown] deletable=true editable=true
# #### 5.1.2 Find Stopping Round with more data

# + deletable=true editable=true
#2. find stopping round
split_index = int(data_train.shape[0]*0.8)
X_train = data_train[:split_index]
X_val = data_train[split_index:]
Y_train = target[:split_index]
Y_val = target[split_index:]

# + deletable=true editable=true
model_xgb1 = xgb.XGBClassifier(max_depth=6, learning_rate=0.05, n_estimators=20000,
                          subsample=0.8, colsample_bytree=0.9, reg_alpha=65,
                          seed=420)

# + deletable=true editable=true
model_xgb1.fit(X_train, Y_train, eval_metric='auc',
          eval_set=[(X_train, Y_train), (X_val, Y_val)],
          early_stopping_rounds=100)

# + [markdown] deletable=true editable=true
# #### 5.1.3 Final Xgboost Model on All Data

# + deletable=true editable=true
model_xgb2 = xgb.XGBClassifier(max_depth=10, learning_rate=0.1, n_estimators=1150,
                          subsample=0.8, colsample_bytree=0.9,
                          seed=420)

# + deletable=true editable=true
model_xgb2.fit(data_train, target, eval_metric='auc',
          eval_set=[(data_train, target)])

# + [markdown] deletable=true editable=true
# ### 5.2 LightGBM Model

# + deletable=true editable=true
import lightgbm as lgb

# + deletable=true editable=true
from sklearn.grid_search import GridSearchCV

grid_model = lgb.LGBMClassifier(reg_alpha=65, max_depth=10, learning_rate=0.1,
                                num_leaves=60, colsample_bytree=0.9, min_child_weight=3,
                                boosting_type='dart', max_bin=255, n_estimators=600,
                                subsample_for_bin=50000, objective=None, min_split_gain=0, 
                                min_child_samples=10, subsample=0.8, 
                                subsample_freq=1, reg_lambda=0, 
                                seed=420)
grid_params = {
    'max_depth':[4,6,8,10],
    'learning_rate':[0.1,0.06,0.03,0.01,0.005,0.001],
}
grid_cv = list(StratifiedKFold(n_splits=2, shuffle=True, random_state=420).split(data_train, target))

grid = GridSearchCV(grid_model, grid_params, scoring='roc_auc',
                    cv=grid_cv, verbose=50)
grid.fit(data_train, target)

# + deletable=true editable=true
grid.best_params_

# + deletable=true editable=true
model_lgb = lgb.LGBMClassifier(reg_alpha=65, max_depth=10, learning_rate=0.1,
                                num_leaves=60, colsample_bytree=0.9, min_child_weight=3,
                                boosting_type='dart', max_bin=255, n_estimators=600,
                                subsample_for_bin=50000, objective=None, min_split_gain=0, 
                                min_child_samples=10, subsample=0.8, 
                                subsample_freq=1, reg_lambda=0, 
                                seed=420)

# + deletable=true editable=true
model_lgb.fit(data_train, target, eval_metric='auc',
              eval_set=[(data_train, target)],
              early_stopping_rounds=100)
#[(X_train, Y_train), (X_val, Y_val)], [(data_train, target)],

# + [markdown] deletable=true editable=true
# ## 6. Make Prediction

# + [markdown] deletable=true editable=true
# ### 6.1 Single Model

# + deletable=true editable=true
Y_pred = model_lgb.predict(data_test)

# + [markdown] deletable=true editable=true
# ### 6.2 Ensemble Models

# + deletable=true editable=true
Y_pred1 = model_xgb2.predict_proba(data_test)
#Y_pred1 = pd.read_csv('pred_xgb.csv').values
# -

output1 = pd.DataFrame(Y_pred1)
output1.to_csv('pred_xgb.csv', index=False)

# + deletable=true editable=true
#Y_pred2 = model_lgb.predict_proba(data_test)
Y_pred2 = pd.read_csv('pred_lgb.csv').values

# + deletable=true editable=true
output2 = pd.DataFrame(Y_pred2)
output2.to_csv('pred_lgb.csv', index=False)

# + deletable=true editable=true
Y_pred = np.apply_along_axis(lambda x: 0 if x[0]>0.5 else 1, 1, 
                             ((Y_pred1 + Y_pred2)/2))

# + [markdown] deletable=true editable=true
# ### 6.3 Save Predictions

# + deletable=true editable=true
output = pd.DataFrame({'project_id': test['project_id'], 'final_status': Y_pred})
output.to_csv('submission26.csv', index=False, columns=['project_id', 'final_status'])

# + [markdown] deletable=true editable=true
# ## ANN

# + deletable=true editable=true
#a = data_train[10000:11000].todense()
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Embedding, LSTM
from keras.layers.advanced_activations import PReLU
# -

data_ann = np.random.random((1, 10000))#data_train[:100].todense()
target_ann = np.random.randint(2, size=(1, 1))

# + deletable=true editable=true
model = Sequential()

# + deletable=true editable=true
model.add(Embedding(10000, output_dim=256))
model.add(LSTM(128))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])
# -

model.fit(data_ann, target_ann, epochs=1, verbose=2)

# +
data_dim = 150000 #192327
timesteps = 8
num_classes = 2

# expected input data shape: (batch_size, timesteps, data_dim)
model = Sequential()
model.add(LSTM(32, return_sequences=True,
               input_shape=(timesteps, data_dim)))  # returns a sequence of vectors of dimension 32
model.add(LSTM(32, return_sequences=True))  # returns a sequence of vectors of dimension 32
model.add(LSTM(32))  # return a single vector of dimension 32
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

# Generate dummy training data
x_train = np.random.random((1000, timesteps, data_dim))
y_train = np.random.randint(num_classes, size=(1000, 1))

# Generate dummy validation data
x_val = np.random.random((100, timesteps, data_dim))
y_val = np.random.randint(num_classes, size=(100, 1))

model.fit(x_train, y_train,
          batch_size=64, epochs=5,
          validation_data=(x_val, y_val))
# -



StarcoderdataJupyterScriptsDedupFiltered