<jupyter_start><jupyter_text>Read DataUse pandas to read csv files and print head<jupyter_code>path_prefix = "./data"
warnings.filterwarnings("ignore")


# Read CSV files to get questions and tags
df_questions = pd.read_csv(path_prefix+"/Questions.csv", encoding="ISO-8859-1")
df_tags = pd.read_csv(path_prefix+"/Tags.csv", encoding="ISO-8859-1", dtype={'Tag': str})


df_questions.head(5)
df_tags = pd.read_csv(path_prefix+"/Tags.csv",encoding="ISO-8859-1",dtype={'Tag':str})
df_tags.head(5)<jupyter_output><empty_output><jupyter_text>Process tagsProcess them tags into something nice to query<jupyter_code>type(df_tags)
type(df_tags['Tag'][0])
# Group tags by id and join them
df_tags['Tag'] = df_tags['Tag'].astype(str)
grouped_tags = df_tags.groupby("Id")['Tag'].apply(lambda tags: ' '.join(tags))
grouped_tags.head(5)
# Reset index for making simpler dataframe
grouped_tags = grouped_tags.reset_index()
a = df_tags.groupby('Id')
b = a['Tag'].apply(lambda tags: ' '.join(tags))
b = b.reset_index()
grouped_tags_final = pd.DataFrame({'Id':b['Id'],'Tags':b['Tag']})
grouped_tags_final.head(5)
# Drop unnecessary columns
df_questions.drop(columns=['OwnerUserId', 'CreationDate', 'ClosedDate'], inplace=True)

# Merge questions and tags into one dataframe
df = df_questions.merge(grouped_tags_final, on='Id')
df.head(5)<jupyter_output><empty_output><jupyter_text>Visualization for score<jupyter_code>sum(df['Score']>5)
type(df['Score'][0])<jupyter_output><empty_output><jupyter_text>Using the sampling to view the distribution of score<jupyter_code>index = np.random.randint(0,1264216,size = 10000)
df['Score'][index]
#直方图📊
plt.hist(df['Score'][index],facecolor='blue',edgecolor='black',alpha=0.35)
plt.xlabel('range')
plt.ylabel('frequency')
plt.title('Frequency distribution histogram')
#my_x_ticks = np.arange(0, 100, 50)

#plt.xticks(my_x_ticks)
plt.show()
plt.boxplot(df['Score'])
plt.ylim(-10, 10)
new_df = df[df['Score']>5]
new_df.head()
new_df.info()
result = new_df['Score'].describe()
pd.DataFrame(result)<jupyter_output><empty_output><jupyter_text>there is no missing values or duplocated values<jupyter_code>print('Duplicate entries:{}'.format(new_df.duplicated().sum()))
new_df.drop_duplicates(inplace=True)<jupyter_output><empty_output><jupyter_text>This is a very good dataset since there are no missing values or duplicated values.<jupyter_code>new_df.drop(columns=['Id','Score'],inplace=True)
new_df.head(10)
# Filter out questions with a score lower than 5
new_df = df[df['Score']>5]

# Split tags in order to get a list of tags
new_df['Tags'] = new_df['Tags'].apply(lambda x: x.split())
all_tags = [item for sublist in new_df['Tags'].values for item in sublist]

flat_list = [item for sublist in new_df['Tags'].values for item in sublist]

keywords = nltk.FreqDist(flat_list)
keywords = nltk.FreqDist(keywords)

# Get most frequent tags
frequencies_words = keywords.most_common(25)
tags_features = [word[0] for word in frequencies_words]
# Drop unnecessary columns at this point
new_df.drop(columns=['Id', 'Score'], inplace=True)
print(tags_features)
hot_tags = [item[0] for item in frequencies_words[:10]]
hot_tags 
hot_tags_nums = [item[1] for item in frequencies_words[:10]]
hot_tags_nums 
#参数一：y轴 参数二：x轴
plt.barh(range(10),hot_tags_nums,height = 0.7,color = 'steelblue',alpha =0.8)
plt.yticks(range(10),hot_tags)
plt.xlim(2000,7000)
for x, y in enumerate(hot_tags_nums):
    plt.text(y + 0.2, x - 0.1, '%s' % y)
plt.xlabel('frequency')
plt.title('The frequency of TOP 10 labels')
plt.show()
keyword = nltk.FreqDist(keywords)
keyword
#most_common()函数 
# most_common(n)按照降序，返回前n项组成的list; n忽略时返回全部
frequencies_words = keyword.most_common(100)
tag_features = [word[0] for word in frequencies_words]
frequencies_words[:10]
tag_features[:10]
fig , ax = plt.subplots(figsize = (15,10))
keyword.plot(100,cumulative = False)
#cumulative：whether to count the cumulative frequenceies
plt.figure(figsize=(10,10))
plt.bar(hot_tags,hot_tags_nums,color='rgb',tick_label=hot_tags)

plt.xlabel('hot tags')
plt.ylabel('the number of hot tags')
plt.title('Frequency of high-frequency tag statistics')
plt.show()
def most_common(tags):
    """Function to check if tag is in most common tag list"""
    tags_filtered = []
    for i in range(0, len(tags)):
        if tags[i] in tags_features:
            tags_filtered.append(tags[i])
    return tags_filtered

# Change Tags column into None for questions that don't have a most common tag
new_df['Tags'] = new_df['Tags'].apply(lambda x: most_common(x))
new_df['Tags'] = new_df['Tags'].apply(lambda x: x if len(x)>0 else None)

# Drop rows that contain None in Tags column
new_df.dropna(subset=['Tags'], inplace=True)
new_df.shape<jupyter_output><empty_output><jupyter_text>Preprocess Data    Remove special characters from title and body    Remove stop words    Remove HTML tags    Convert characters to lowercase    Lemmatize the words<jupyter_code># Filter out HTML
new_df['Body_lxml'] = new_df['Body'].apply(lambda x: BeautifulSoup(x, 'lxml').get_text()) 
token = ToktokTokenizer()
lemma = WordNetLemmatizer()
stop_words = set(stopwords.words("english"))

def strip_list_noempty(mylist):
    newlist = (item.strip() if hasattr(item, 'strip') else item for item in mylist)
    return [item for item in newlist if item != '']

def removeStopWords(text):
    words = token.tokenize(text)
    filtered = [w for w in words if not w in stop_words]
    return ' '.join(map(str, filtered))

def removePunctuation(text):
    punct = '!"$%&\'()*,./:;<=>?@[\\]^_`{|}~'
    words=token.tokenize(text)
    punctuation_filtered = []
    regex = re.compile('[%s]' % re.escape(punct))
    remove_punctuation = str.maketrans(' ', ' ', punct)
    for w in words:
        if w in tags_features:
            punctuation_filtered.append(w)
        else:
            punctuation_filtered.append(regex.sub('', w))
  
    filtered_list = strip_list_noempty(punctuation_filtered)
        
    return ' '.join(map(str, filtered_list))

def lemmatizeWords(text):
    words=token.tokenize(text)
    listLemma=[]
    for w in words:
        x=lemma.lemmatize(w, pos="v")
        listLemma.append(x.lower())
    return ' '.join(map(str, listLemma))

# Remove stopwords, punctuation and lemmatize for text in body
new_df['Body_punctuation'] = new_df['Body_lxml'].apply(lambda x: removePunctuation(x))
new_df.head(10)
new_df['Body_stopword'] = new_df['Body_punctuation'].apply(lambda x: removeStopWords(x))
new_df['Body_lemmatize'] = new_df['Body_stopword'].apply(lambda x: lemmatizeWords(x))
new_df.head(10)
new_df['Final_Body'] = new_df['Body_lemmatize']
new_df['Final_Body'].head(5)
# Remove stopwords, punctuation and lemmatize for title. Also weight title 3 times
new_df['Title_str'] = new_df['Title'].apply(lambda x: str(x)) 
new_df['Title_punctuation'] = new_df['Title_str'].apply(lambda x: removePunctuation(x)) 
new_df['Title_stopword'] = new_df['Title_punctuation'].apply(lambda x: removeStopWords(x)) 
new_df['Title_lemmatize'] = new_df['Title_stopword'].apply(lambda x: lemmatizeWords(x)) 
new_df['Title_weight'] = new_df['Title_lemmatize'].apply(lambda x: ' '.join(x.split()*3))
new_df['Title'] = new_df['Title_weight']
new_df['Title']
#new_df.to_csv("./data/processed_question.csv")<jupyter_output><empty_output><jupyter_text>-EDA<jupyter_code>no_topics=20
text = new_df['Final_Body']
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer_train = TfidfVectorizer(analyzer = 'word',min_df=0.0,max_df=1.0,strip_accents=None,
                                  encoding='utf-8', preprocessor=None,token_pattern=r"(?u)\S\S+",
                                  max_features=1000
                                 )
TF_IDF_matrix = vectorizer_train.fit_transform(text)
TF_IDF_matrix.shape
weight = TF_IDF_matrix.toarray()
weight.shape
import sys
import numpy
numpy.set_printoptions(threshold=sys.maxsize)
print(weight[0])
print(TF_IDF_matrix)
from sklearn.decomposition import LatentDirichletAllocation
lda = LatentDirichletAllocation(n_components=no_topics,max_iter=5,learning_method='online',learning_offset=50,random_state=11).fit(TF_IDF_matrix)
def display_topics(model,feature_names,no_stop_words):
    for topic_idx, topic in enumerate(model.components_):
        print("------------------------------------------")
        print("Topic %d:"%(topic_idx))
        print(" ".join([feature_names[i] for i in topic.argsort()[:-no_top_words-1:-1]]))
        print("------------------------------------------")
no_top_words =10
display_topics(lda,vectorizer_train.get_feature_names(),no_top_words)<jupyter_output>------------------------------------------
Topic 0:
table select query row database sql column id value mysql
------------------------------------------
------------------------------------------
Topic 1:
use android application app find would like server web know
------------------------------------------
------------------------------------------
Topic 2:
devices eandroidruntime mobile linearlayout red dialog extra target 05 19
------------------------------------------
------------------------------------------
Topic 3:
file error project run use install build command try --
------------------------------------------
------------------------------------------
Topic 4:
pdf can explain book socket engine someone please block tell
------------------------------------------
------------------------------------------
Topic 5:
class public object string method new return type use value
------------------------------------------
------------------------------------------
Topic 6:
script ht[...]<jupyter_text>Classifier implementation<jupyter_code>from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from scipy.sparse import hstack

# Define X, y
X1 = new_df['Final_Body']
X2 = new_df['Title']
y = new_df['Tags']
print(len(X1), len(X2), len(y))

# Define multilabel binarizer
multilabel_binarizer = MultiLabelBinarizer()
y_bin = multilabel_binarizer.fit_transform(y)


vectorizer_X1 = TfidfVectorizer(analyzer = 'word',
                                       min_df=0.0005,
                                       max_df = 1.0,
                                       strip_accents = None,
                                       encoding = 'utf-8', 
                                       ngram_range = (1, 3),
                                       preprocessor=None,
                                       token_pattern=r"(?u)\S\S+",
                                       max_features=35000)

vectorizer_X2 = TfidfVectorizer(analyzer = 'word',
                                       min_df=0.0,
                                       max_df = 1.0,
                                       strip_accents = None,
                                       encoding = 'utf-8', 
                                       ngram_range = (1, 3),
                                       preprocessor=None,
                                       token_pattern=r"(?u)\S\S+",
                                       max_features=35000)

X1_tfidf = vectorizer_X1.fit_transform(X1)
X2_tfidf = vectorizer_X2.fit_transform(X2)

# Stack X1 and X2 into X_tfidf
X_tfidf = hstack([X1_tfidf,X2_tfidf])

# Split training and test data    
X_train, X_test, y_train, y_test = train_test_split(X_tfidf, y_bin, test_size = 0.2, random_state = 0)
# Using Label Powerset
from sklearn.metrics import accuracy_score
from sklearn.svm import LinearSVC
from sklearn.metrics import hamming_loss
from sklearn.metrics import f1_score
from skmultilearn.problem_transform import LabelPowerset
from sklearn.linear_model import SGDClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import jaccard_score
from sklearn import model_selection
from sklearn.metrics import make_scorer
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
svc = LinearSVC()
sgd = SGDClassifier(n_jobs=-1)

def print_score(y_pred, clf):
    print("Clf: ", clf.__class__.__name__)
    print("Accuracy score: {}".format(accuracy_score(y_test, y_pred)))
    print("Recall score: {}".format(recall_score(y_true=y_test, y_pred=y_pred, average='weighted')))
    print("Precision score: {}".format(precision_score(y_true=y_test, y_pred=y_pred, average='weighted')))
    print("Hamming loss: {}".format(hamming_loss(y_pred, y_test)*100))
    print("F1 score: {}".format(f1_score(y_pred, y_test, average='weighted')))
    print("---")    

clf = LabelPowerset(svc)

clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print_score(y_pred, clf)

kfold = KFold(n_splits=5)
X_sparse = X_tfidf.tocsr()

scores = []

for train_indices, test_indices in kfold.split(X_sparse, y_bin):
    clf.fit(X_sparse[train_indices], y_bin[train_indices])
    print(clf.score(X_sparse[test_indices], y_bin[test_indices]))
    scores.append(clf.score(X_sparse[test_indices], y_bin[test_indices]))

print(sum(scores)/len(scores))
# Using Classifier Chains
from sklearn.multioutput import ClassifierChain
import numpy as np


chains = [ClassifierChain(svc, order='random', random_state=i)
          for i in range(10)]

for chain in chains:
    chain.fit(X_train, y_train)

Y_pred_chains = np.array([chain.predict(X_test) for chain in
                          chains])

Y_pred_ensemble = Y_pred_chains.mean(axis=0)
ensemble_accuracy_score = accuracy_score(y_test, Y_pred_ensemble >= .5)
ensemble_recall_score = recall_score(y_test, Y_pred_ensemble >= .5, average='weighted')
ensemble_precision_score = precision_score(y_test, Y_pred_ensemble >= .5, average='weighted')
ensemble_f1_score = f1_score(y_pred, Y_pred_ensemble >= .5, average='weighted')
hamm = hamming_loss(Y_pred_ensemble >= .5, y_test)*100
print(ensemble_accuracy_score, ensemble_recall_score, ensemble_precision_score, ensemble_f1_score, hamm)
# Using Binary Relevance
from skmultilearn.problem_transform import BinaryRelevance
from sklearn.naive_bayes import GaussianNB

# initialize binary relevance multi-label classifier
# with a gaussian naive bayes base classifier
classifier = BinaryRelevance(svc)

# train
classifier.fit(X_train, y_train)

# predict
predictions = classifier.predict(X_test)

print_score(predictions, classifier)<jupyter_output>Clf:  BinaryRelevance
Accuracy score: 0.5623807706982068
Recall score: 0.6469784391132706
Precision score: 0.8453964177308024
Hamming loss: 2.315528424265547
F1 score: 0.7474982642210757
---
StarcoderdataJupyterStructuredCleanDedup