# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# 数据文件格式json分析  
# {"c":"1415.59,13369599,1,25,57b8cce4,1381770056","m":"最后的福利~"}  
# 对应内容  
# {"c": "播放时间,颜色,模式,字号,uid,发送时间", "m": "弹幕内容"}  
# C:  
# dtTime = 播放时间:按秒计  
# rgb = 颜色:十进制颜色  
# danmu_model = 模式:1-3:滚动弹幕4:底端弹幕5:顶端弹幕7:高级弹幕  
# font = 字号:十进制整数 1-99  
# userid = uid:  
# stamp = 发送时间:时间戳  
# m:  
# message = 弹幕内容  

import pandas as pd
from pandas import Series,DataFrame
import numpy as np
import time
import matplotlib.pyplot as plt
# %matplotlib inline

# 导入弹幕数据文件,并按照json内容 划分为列

list_episode = []
for i in range(3,142):
    df = pd.read_json('../data/hxh/%s.json'%(i),encoding='utf-8')
    df = df.rename(columns=lambda x:x.replace('m', 'message'))
    df['dtTime'],df['rgb'],df['danmu_model'],df['font'],df['userid'],df['stamp'] = df['c'].str.split(',',6).str
    df['episode'] = i
    df = df.drop(['c'],axis = 1)
    list_episode.append(df)
data = pd.concat(list_episode,ignore_index=True)
data

# 查看是否有空值

data.isnull().sum().any()

# 查看数据概况

data.shape

data.head()


# 提取时间戳里的 小时 年 月 生成新的列合并

def timechange(times):
    time_local = time.localtime(int(times))
    return time_local.tm_hour


data['hour']= data['stamp'].map(timechange)


def timechangeyear(times):
    time_local = time.localtime(int(times))
    return time_local.tm_year


data['year']= data['stamp'].map(timechangeyear)


def timechangemon(times):
    time_local = time.localtime(int(times))
    return time_local.tm_mon


data['mon']= data['stamp'].map(timechangemon)

data

# 查看弹幕总数

#弹幕总数
data.shape[0]

# 解决plt绘图中文问题

from pylab import mpl
mpl.rcParams['font.sans-serif'] = ['SimHei'] # 指定默认字体：解决plot不能显示中文问题
mpl.rcParams['axes.unicode_minus'] = False    

# 绘图 查看弹幕密度情况 用户发送弹幕数量情况

#x集 y弹幕量    bar+线
data1 = data['episode'].value_counts()
data1 = data1.sort_index()
data1_index = np.array(data1.index)
data1_index
data1_value = np.array(data1.values)
data1_value
plt.bar(data1_index, data1_value, 0.5)
plt.xlabel('集数', size=30)
plt.ylabel('弹幕数', size=30)
plt.plot(data1_index, data1_value, c='red')

#用户弹幕量排名  
data_damperuser = data['userid'].value_counts()
data_damperuser_hbar = data_damperuser.values
plt.bar([x for x in range(0,10)],data_damperuser_hbar[:10],1)
plt.grid(True,lw=1,mfc='green')


# 发现第一名的用户发了30000多发弹幕,远超其他人,考虑为异常,进行检查

#看看第一名
data['userid'].value_counts()

# 确定问题任务id,及信息,发现大量弹幕是同一个时间点发送的,判定为异常情况

cond_usermax = data['userid']=='0bc076ae'
data[cond_usermax]

#查看异常弹幕人物的发弹幕分布
data[cond_usermax]['episode'].value_counts()

#删除这个异常的
data = data.drop(list(data[cond_usermax].index))

#x集 y弹幕量    bar+线
data1 = data['episode'].value_counts()
data1 = data1.sort_index()
data1_index = np.array(data1.index)
data1_index
data1_value = np.array(data1.values)
data1_value
plt.bar(data1_index,data1_value,0.5)
plt.xlabel('集数',size=30)
plt.ylabel('弹幕数',size=30)
plt.plot(data1_index,data1_value,c='red')

# 除去问题id弹幕后,最后几级数据异常,应当删除

#去除有问题的集数
cond_err_episode = data['episode']>135
data = data.drop(list(data[cond_err_episode].index))
data.shape

#x集 y弹幕量    bar+线
data1 = data['episode'].value_counts()
data1 = data1.sort_index()
data1_index = np.array(data1.index)
data1_index
data1_value = np.array(data1.values)
data1_value
plt.figure(figsize=(10,5))
plt.bar(data1_index,data1_value,0.5)
plt.xlabel('集数',size=30)
plt.ylabel('弹幕数',size=30)
plt.plot(data1_index,data1_value,c='red')

# 查看弹幕最少的是哪几集

data['episode'].value_counts().tail(10)

# 87集 436弹幕 分析:
# 10-13集:
# 31-38集:

data_damperuser = data['userid'].value_counts()
data_damperuser_hbar = data_damperuser.values
ing=range(11)
plt.bar(top10_index,data_damperuser_hbar[:10],0.5)
plt.xticks(ing, ing, rotation=30)
plt.xlabel('用户弹幕量排名TOP10',size=20)
plt.ylabel('弹幕数',size=20)

# 查看用户整部动画发送多少条弹幕的分布

#用户弹幕数量百分比 饼图
data_user_dam = DataFrame({'count':data_damperuser.values,'userid':data_damperuser.index})
data_user_dam


def danmu_length_pie(data_user_dam):   
    sumall = data_user_dam.shape[0]
    a1 = (data_user_dam['count']==1).sum()
    a2_5 = ((data_user_dam['count']>1)&(data_user_dam['count']<=5)).sum()
    a6_10 = ((data_user_dam['count']>5)&(data_user_dam['count']<=10)).sum()
    a11_50 = ((data_user_dam['count']>10)&(data_user_dam['count']<=50)).sum()
    a51_100 = ((data_user_dam['count']>50)&(data_user_dam['count']<=100)).sum()
    a101_500 = ((data_user_dam['count']>100)&(data_user_dam['count']<=500)).sum()
    a501 = (data_user_dam['count']>500).sum()
    li = [a1,a2_5,a6_10,a11_50,a51_100,a101_500,a501]
    xp=[]
    for i in li:
        i = float(i)
        t = (i/sumall)
        xp.append(t)
    plt.figure(figsize=(6,9))
    labels = [u'1个', u'2-5个', u'6-10个', u'11-50个',u'51-100个',u'101-500个',u'500个以上' ]
    sizes = xp
    plt.axis('equal')
    plt.title(u'全集用户弹幕数量分布图',loc='left')
    plt.pie(sizes,  labels=labels,labeldistance=1.1,autopct = '%2.2f%%', startangle = 90, pctdistance = 0.5,explode=[0,0,0,0,0.5,1,1.5],shadow=True)
    plt.show()
    plt.close()



danmu_length_pie(data_user_dam)

data.head()

#每集弹幕分布 x时间 y弹幕量
data.shape

# 查看平均每集的弹幕密度分布

cond4 = (data.episode==33)
max3 = data[cond4]['dtTime'].max()
min3 = data[cond4]['dtTime'].min()
display(max3,min3)


def dtTimemin(dttime):
    return float(dttime)//60
data['dtTimemin']=data['dtTime'].map(dtTimemin)

data['dtTimemin'].value_counts()

# 一般每集时间为20分钟左右, 41446分钟为异常值,查看后删除

conds = data['dtTimemin']==41446.0
dropindex = data[conds].index[0]
data = data.drop(dropindex)

# 查看41分钟的弹幕,判断为视频进度条的问题,可以作为bug问题等反馈给相关部门,改善用户体验

conds2 = data['dtTimemin']==41.0
data[conds2]

data = data.drop(data[conds2].index[0])

data_meanday = data['dtTimemin'].value_counts().sort_index()

data_meanday

plt.xlabel('动画播放分钟',size=30)
plt.ylabel('弹幕数',size=30)
plt.plot(np.array(data_meanday.index),np.array(data_meanday.values),c='red')

# 上图说明 大家喜欢再开始播放和oped的时候发送弹幕 20分钟以后的下降 是因为41分钟的进度条bug

# 下面看看发送弹幕的观众,喜欢在几点钟观看视频

# x小时  y弹幕数量
data_24 = data['hour'].value_counts().sort_index()
data_24

plt.xlabel('发弹幕时间(小时)',size=30)
plt.ylabel('弹幕数',size=30)
plt.plot(np.array(data_24.index),np.array(data_24.values),c='red')

# 可以看出 大部分人睡眠时间在5-6点周围 大概1-9点的范围. 当中的突起,代表了午间休息和午饭时间, 17点以后逐渐增加进入高峰,代表下班/放学以后

# 下面制作一个词云,分析大家的弹幕中最关注的词

testword = ' '.join(data.message.values)

import jieba  # 分词包
from wordcloud import WordCloud, ImageColorGenerator    # 词云包
from scipy.misc import imread
segtests = jieba.cut(testword)

segment = []
for seg in segtests:
    if len(seg) > 1 and seg != '\r\n':
        segment.append(seg)

# 去停用词(文本去噪)
words_df = pd.DataFrame({'segment': segment})
# 字典中的keys就是DataFrame里面的columns，但是没有index的值，所以需要自己设定，不设定默认是从零开始计数。
words_df.head()

words_df.shape

#去停用词
stopwords = pd.read_csv("../data/stopwords.txt", index_col=False,quoting=3, sep='\t', names=['stopword'], encoding="utf8")
words_df = words_df[~words_df.segment.isin(stopwords.stopword)]

# 词汇频率表
words_stat = words_df.groupby(by=['segment'])['segment'].agg({"count": np.size})
words_stat = words_stat.reset_index().sort_values(by="count", ascending=False)

words_stat.head(10)

content = ' '.join(words_stat.head(40).segment.values)

## 自定义词云背景
#bimg = imread('heart.jpeg')
# #bimgColors = ImageColorGenerator(bimg)
wordcloud = WordCloud(font_path='simhei.ttf', background_color="white",max_words=40).generate(content)
plt.axis("off")
plt.imshow(wordcloud)
plt.show()

StarcoderdataJupyterScriptsDedupFiltered