<jupyter_start><jupyter_text><jupyter_code>#Major libraries
import pandas as pd 
import numpy as np
import matplotlib.pyplot as plt 
import seaborn as sns
import copy as cp
from datetime import datetime
import warnings
warnings.filterwarnings("ignore")
#Cross Validattion tools 
from sklearn import datasets
from sklearn.feature_selection import RFE
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
#Import Supervised Learning models 
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import Ridge
from sklearn.linear_model import BayesianRidge
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn import svm
import xgboost as xgb<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code># connect to google drive and extract csv files
metrics= pd.read_csv('/content/drive/My Drive/Projects/FINESSE/post_metrics_and_comments.csv')
metrics.head()
# make a copy of original drive for modifications
new_metrics = cp.deepcopy(metrics)
# remove Null values
new_metrics.dropna(axis= 1, inplace= True)
new_metrics.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 11637 entries, 0 to 11636
Data columns (total 9 columns):
 #   Column                 Non-Null Count  Dtype 
---  ------                 --------------  ----- 
 0   post_url               11637 non-null  object
 1   username               11637 non-null  object
 2   date_posted            11637 non-null  object
 3   img_urls               11637 non-null  object
 4   max_likes              11637 non-null  int64 
 5   max_comments           11637 non-null  int64 
 6   followers              11637 non-null  int64 
 7   following              11637 non-null  int64 
 8   concatenated_comments  11637 non-null  object
dtypes: int64(4), object(5)
memory usage: 818.4+ KB<jupyter_text>Since we have the maximum number of likes for each post(max_likes), the main criteria for trending post can be ** the number of likes divided by mean(average) of likes for each username assigned to all the post_url**. it also can simply called **weighted_mean_likes**. Therefore we use feature engineering to extract features that help our regression model to predict the **weighted_mean_likes** more accuretely.   Feature Engineering  Median, means and weighted mean of likes and commentsAdding mean of likes/comments and weighted mean of likes/comments as 4 new features<jupyter_code>def weighted_mean_columns(dataframe, col, new_col):
  mean = dataframe.groupby('username')[col].mean()

  # def weighted_average(matrix, col_name)
  hash ={}
  for idx in range(len(mean)):
    hash[mean.index[idx]] = mean[idx]

  dataframe[new_col] = None
  for idx in range(len(dataframe)):
    username = dataframe.username[idx]
    dataframe[new_col][idx] = round(dataframe[col][idx].astype(float) / hash[username] , 6 )

def mean(dataframe, col, new_col):
  mean = dataframe.groupby('username')[col].mean()
  hash = {}
  for idx in range(len(mean)):
    hash[mean.index[idx]] = mean[idx]
  dataframe[new_col] = None
  for idx in range(len(dataframe)):
    username = dataframe.username[idx]
    dataframe[new_col][idx] = round(hash[username] , 6)

def median(dataframe, col, new_col):
  median = dataframe.groupby('username')[col].median()
  hash = {}
  for idx in range(len(median)):
    hash[median.index[idx]] = median[idx]
  
  dataframe[new_col] = None
  for idx in range(len(dataframe)):
    username = dataframe.username[idx]
    dataframe[new_col][idx] = round(hash[username] , 6)


# running functions
weighted_mean_columns(new_metrics, 'max_likes', 'weighted_mean_likes')
weighted_mean_columns(new_metrics, 'max_comments', 'weighted_mean_comments')
mean(new_metrics, 'max_likes', 'mean_likes')
mean(new_metrics, 'max_comments', 'mean_comments')
median(new_metrics, 'max_likes', 'median_likes')


# changing the data type from object to float 
new_metrics['weighted_mean_likes'] = new_metrics['weighted_mean_likes'].astype(float)
new_metrics['weighted_mean_comments'] = new_metrics['weighted_mean_comments'].astype(float)
new_metrics['mean_likes'] = new_metrics['mean_likes'].astype(float)
new_metrics['mean_comments'] = new_metrics['mean_comments'].astype(float)
new_metrics['median_likes'] = new_metrics['median_likes'].astype(float)

new_metrics.head()
# remove all the null values
new_metrics.dropna(axis=0, inplace=True)
new_metrics.isnull().sum()<jupyter_output><empty_output><jupyter_text>Adding days of week(Monday, Tuesday, ...)<jupyter_code>#find the start and end date of data
print(new_metrics['date_posted'].min())
print(new_metrics['date_posted'].max())
# Adding days as a features
new_metrics['Monday'] = 0 
new_metrics['Tuesday'] = 0 
new_metrics['Wednesday'] = 0 
new_metrics['Thursday'] = 0 
new_metrics['Friday'] = 0 
new_metrics['Saturday'] = 0 
new_metrics['Sunday'] = 0 

for i, dates in enumerate(new_metrics['date_posted']):
  if i == 0:
    start_day = datetime(2020,4,1)
  date_time = datetime.strptime(dates, f'%Y-%m-%d')
  n = (date_time - start_day).days %7
  if n==0:
    new_metrics['Wednesday'][i] = 1
  elif n==1:
    new_metrics['Thursday'][i] = 1
  elif n==2:
    new_metrics['Friday'][i] = 1
  elif n==3:
    new_metrics['Saturday'][i] = 1
  elif n==4:
    new_metrics['Sunday'][i] = 1
  elif n==5:
    new_metrics['Monday'][i] = 1
  elif n==6:
    new_metrics['Tuesday'][i] =1

new_metrics.head()<jupyter_output><empty_output><jupyter_text>Adding days that days that post was in top trending hashtags:<jupyter_code>hashtags = pd.read_csv('/content/drive/My Drive/Projects/FINESSE/hashtag_top_appearances.csv')
hashtags.head()
hashtags['post_url'] = hashtags['post_url'].astype(str)
hashtags.info()
new_metrics['days_in_hashtag_top_section'] = 0 
for idx, hashtag1 in enumerate(new_metrics['post_url']):
  for idx2, hashtag2 in enumerate(hashtags['post_url']):
    if hashtag2 == hashtag1:
      new_metrics['days_in_hashtag_top_section'][idx] = hashtags['days_in_hashtag_top_section'][idx2]
      break
    
new_metrics.head()<jupyter_output><empty_output><jupyter_text>Adding number of posts for each username<jupyter_code>new_metrics['number_of_posts'] = 0
dummy = new_metrics.groupby('username')['post_url'].count()
for idx, line in enumerate(new_metrics['username']):
  new_metrics['number_of_posts'][idx] = dummy[line]
new_metrics.head()<jupyter_output><empty_output><jupyter_text>Correlation Analysis Find the correlation of other parameters to<jupyter_code>correlation = new_metrics.corr(method='spearman')
columns = correlation.nlargest(10, 'weighted_mean_likes').index
correlation_map = np.corrcoef(new_metrics[columns].values.T)
sns.set(font_scale=0.6)
sns.set(rc={'figure.figsize':(11.7,8.27)})
heatmap = sns.heatmap(correlation_map, cbar=True, annot=True, square=True, fmt='.2f', yticklabels=columns.values, xticklabels=columns.values)
plt.show()<jupyter_output><empty_output><jupyter_text>The most significant correlations are listed below:<jupyter_code>correlations=new_metrics.corr()
attrs = correlations.iloc[:-1,:-1] # all except target

threshold = 0.5
important_corrs = (attrs[abs(attrs) > threshold][attrs != 1.0]).unstack().dropna().to_dict()

unique_important_corrs = pd.DataFrame(
    list(set([(tuple(sorted(key)), important_corrs[key]) \
    for key in important_corrs])), 
        columns=['Attribute Pair', 'Correlation'])

    # sorted by absolute value
unique_important_corrs = unique_important_corrs.loc[
    abs(unique_important_corrs['Correlation']).argsort()[::-1]]

unique_important_corrs
new_metrics[['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday','Saturday','Sunday','weighted_mean_likes']].groupby(['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday','Saturday','Sunday'],
as_index=False).mean().sort_values(by='Sunday', ascending=False)
sns.distplot(new_metrics['weighted_mean_likes'], color="r", kde=False)
plt.title("Distribution of Likes Divided by Mean")
plt.ylabel("Number of Occurences")
plt.xlabel("Number of Likes Divided by Mean Likes for each user");<jupyter_output><empty_output><jupyter_text>Return unbiased skew over requested axis. $<jupyter_code># Measuring the skewness of the weighted_mean_likes column which 
# Skewness is the indication of symmetry in the distribution and the equatuin is E((X-mean)/(standard_deviation)**3)
new_metrics['weighted_mean_likes'].skew()
# measuring the Kurtosis of the weighted_mean_likes column.
# Kurtosis measure the strength of long tail in the distribution and the equation is  E((X-mean)/(standard_deviation)**4)
new_metrics['weighted_mean_likes'].kurt()
plt.scatter(new_metrics['followers'], new_metrics["weighted_mean_likes"], color='orange')
plt.title("Number of Likes Divided by Mean Likes vs Number of Followers")
plt.xlabel("Number of Followers")
plt.ylabel("Number of Likes Divided by Mean Likes");<jupyter_output><empty_output><jupyter_text>Training and Predictions with the model<jupyter_code>new_metrics.drop(['post_url','username', 'date_posted', 'img_urls', 'concatenated_comments'], axis=1, inplace= True)
new_metrics.sort_values(by = 'weighted_mean_likes', ascending = True, inplace= True)
new_metrics.tail()
new_metrics_original = cp.deepcopy(new_metrics)
X, y = new_metrics.drop('weighted_mean_likes',axis=1), new_metrics.weighted_mean_likes
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=44)
# X.info()
models = []
models.append(('GBM',GradientBoostingRegressor()))
models.append(('LR',LinearRegression()))
models.append(('BYR', BayesianRidge()))
models.append(("KNN", KNeighborsRegressor()))
models.append(('XGB', xgb.XGBRegressor()))

for name,model in models:
    kfold = KFold(n_splits=2, random_state=22)
    cv_result = cross_val_score(model,X_train,y_train, cv = kfold, scoring = "r2")
    print(name, cv_result)
X, y = new_metrics_original.drop('weighted_mean_likes', axis=1), new_metrics_original.weighted_mean_likes
X_train2, X_test2, y_train2, y_test2 = train_test_split(X, y, test_size=0.2, shuffle = False)
model = xgb.XGBRegressor()
model.fit(X_train,y_train)
predictions = model.predict(X_test2)
y_train.head()
compare = pd.DataFrame({'Prediction': predictions, 'Ground Truth' : y_test2})
print(compare.tail(10))
pred= sorted(predictions, reverse= True)
print(pred[:10])<jupyter_output>[19:15:42] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
       Prediction  Ground Truth
4048     5.976807      5.518340
10091    3.592964      5.602607
10917    5.343872      5.697391
3451     4.395563      5.768078
5364     5.564166      5.918774
1706     5.726154      6.253734
3868     5.595409      6.448727
2911     6.724216      7.675192
3120     7.076157      9.623617
3406     9.577948     11.498964
[9.577948, 7.076157, 6.7242155, 5.9768066, 5.7261543, 5.595409, 5.5641656, 5.343872, 4.4611506, 4.452263]<jupyter_text>Top 5 predicted post are as follows:<jupyter_code>metrics.loc[[3406, 3120, 2911, 4048, 1706], 'post_url']<jupyter_output><empty_output>