# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Stack Exchange 
# Data Science Stack Exchange is a question and answer site for Data science professionals, Machine Learning specialists, and those interested in learning more about the field. It only takes a minute to sign up.

# review, types

# SELECT Id, PostTypeId, CreationDate, Score, ViewCount, Tags, AnswerCount, FavoriteCount
#   FROM Posts
#   WHERE CreationDate >= DATEADD(DAY, 1, '1/1/2019') AND
#        CreationDate < DATEADD(DAY, 1, '1/1/2020')

import pandas as pd
questions2019 = pd.read_csv('2019_questions.csv')

questions2019.info()

pd.isna(questions2019['FavoriteCount']).sum()

questions2019

# NaN should be changed to zeroes

# Tags contain multiple values so maybe we should separate them or atleat count how many there a

questions2019=questions2019.fillna(0)

pd.isna(questions2019['FavoriteCount']).sum()

questions2019['Tags']=questions2019['Tags'].str.replace('><', ',').str.replace('<','').str.replace('>','')

questions2019['CreationDate']=questions2019['CreationDate'].astype('datetime64[ns]')

questions2019['FavoriteCount']=questions2019['FavoriteCount'].astype('int')

questions2019.info()

tag_list = questions2019['Tags'].tolist() 

# +
split_data = questions2019['Tags'].str.split(",")
questions2019['Tags2'] = split_data
#data = split_data.to_list()

# expand df.tags into its own dataframe
tags = questions2019['Tags2'].apply(pd.Series)

# rename each variable is tags
tags = tags.rename(columns = lambda x : 'tag_' + str(x))



# join the tags dataframe back to the original dataframe
questions2019=pd.concat([questions2019[:], tags[:]], axis=1)
# -

a=questions2019['tag_0'].tolist()
a.extend(questions2019['tag_1'].tolist())
a.extend(questions2019['tag_2'].tolist())
a.extend(questions2019['tag_3'].tolist())
a.extend(questions2019['tag_4'].tolist())

tagdf = pd.DataFrame(a, columns =['Count']) 

unique_tags = tagdf['Count'].unique()

t = tagdf['Count'].value_counts()
tags=pd.DataFrame(t)

tags.index

# +
most_used=tags.head(20)
import matplotlib.pyplot as plt

# %matplotlib inline
most_used.plot(kind="barh")


# +

tag_view_count = dict()

for idx, tagsx in enumerate(questions2019["Tags"]):
    for tag in tagsx:
        if tag in tag_view_count:
            tag_view_count[tag] += questions2019["ViewCount"].iloc[idx]
        else:
            tag_view_count[tag] = 1
            
tag_view_count = pd.DataFrame.from_dict(tag_view_count, orient="index")
tag_view_count.rename(columns={0: "ViewCount"}, inplace=True)

most_viewed = tag_view_count.sort_values(by="ViewCount").tail(20)
# -

tags.index

t.index

questions2019

# +
tag_views = {}

for idx, tagx in enumerate(questions2019["tag_0"]):
    if tagx in tag_views:
        tag_views[tagx] += questions2019.ViewCount.iloc[idx]
    else:
        tag_views[tagx] = 1
for idx, tagx in enumerate(questions2019["tag_1"]):
    if tagx in tag_views:
        tag_views[tagx] += questions2019.ViewCount.iloc[idx]
    else:
         tag_views[tagx] = 1
for idx, tagx in enumerate(questions2019["tag_2"]):
    if tagx in tag_views:
        tag_views[tagx] += questions2019.ViewCount.iloc[idx]
    else:
        tag_views[tagx] = 1
for idx, tagx in enumerate(questions2019["tag_3"]):
    if tagx in tag_views:
        tag_views[tagx] += questions2019.ViewCount.iloc[idx]
    else:
        tag_views[tagx] = 1
for idx, tagx in enumerate(questions2019["tag_4"]):
    if tagx in tag_views:
        tag_views[tagx] += questions2019.ViewCount.iloc[idx]
    else:
        tag_views[tagx] = 1

# +
tag_view_count = pd.DataFrame.from_dict(tag_views, orient="index")

tag_view_count=tag_view_count.drop(tag_view_count.index[0])

 
tag_view_count.rename(columns={0: "ViewCount"}, inplace=True)

most_viewed = tag_view_count.sort_values(by="ViewCount").tail(20)

most_viewed.plot(kind="barh")
# -

in_used = pd.merge(most_used, most_viewed, how="left", left_index=True, right_index=True)

in_used

most_used

for m in most_used.index:
    print(m)

# creating a list of all tags:
all_tags = list(tag_view_count.index)


# +

correlation = pd.DataFrame(index=all_tags, columns = all_tags)
correlation["None"] = 0
correlation.loc["None"] = 0
correlation.fillna(0, inplace=True)

correlation
# -

for ts in questions2019['Tags'].str.split(','):
    for t1 in correlation.index:
        for t2 in correlation.columns:
            if (t1 in ts) and (t2 in ts):
                if correlation.loc[t1,t2] == 0:
                    correlation.loc[t1,t2] = 1
                else:
                    correlation.loc[t1,t2] += 1

correlation

most_used_corr=correlation() 



# +

corr_most_used = correlation.loc[most_used.index, most_used.index]
corr_most_used
# -

corr_most_used.corr(method ='pearson') 

# Tensorflow allways with machine learning

StarcoderdataJupyterScriptsDedupFiltered