# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# Understand dataset
# -

import pandas as pd
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
import seaborn as sns
import numpy as np
import json
import warnings
warnings.filterwarnings('ignore')
import base64
import io
from imageio import imread
import codecs
from IPython.display import HTML
import re

movies = pd.read_csv('tmdb_5000_movies.csv')

mov = pd.read_csv('tmdb_5000_credits.csv')

# +
# 第一个文件Json格式数据提取

# +
movies['genres']=movies['genres'].apply(json.loads)
for index,i in zip(movies.index,movies['genres']):
    list1=[]
    for j in range(len(i)):
        list1.append((i[j]['name']))
    movies.loc[index,'genres']=str(list1)

movies['keywords']=movies['keywords'].apply(json.loads)
for index,i in zip(movies.index,movies['keywords']):
    list1=[]
    for j in range(len(i)):
        list1.append((i[j]['name']))
    movies.loc[index,'keywords']=str(list1)

# movies['production_companies']=movies['production_companies'].apply(json.loads)
# for index,i in zip(movies.index,movies['production_companies']):
#     list1=[]
#     for j in range(len(i)):
#         list1.append((i[j]['id']))
#     movies.loc[index,'production_companies']=str(list1)
      
movies['production_countries']=movies['production_countries'].apply(json.loads)
for index,i in zip(movies.index,movies['production_countries']):
    list1=[]
    for j in range(len(i)):
        list1.append((i[j]['iso_3166_1']))
    movies.loc[index,'production_countries']=str(list1)
# -

movies['production_companies']=movies['production_companies'].apply(json.loads)
def production_company(x):
    list1 = []
    for i in x:
        if(i['id'] not in list1):
            list1.append(i['id'])
    return list1
movies['production_companies']=movies['production_companies'].apply(production_company)

movies.head()



# +
# 把字符串转成list
# -

movies['genres']=movies['genres'].str.strip('[]').str.strip().str.replace("'",'')
movies['genres']=movies['genres'].str.split(', ')
movies['keywords'] = movies['keywords'].str.strip('[]').str.strip().str.replace("'",'')
movies['keywords']=movies['keywords'].str.split(', ')
# movies['production_companies']=movies['production_companies'].str.strip('[]').str.strip().str.replace("'",'')
# movies['production_companies']=movies['production_companies'].str.split(', ')
movies['production_countries']=movies['production_countries'].str.strip('[]').str.strip().str.replace("'",'')
movies['production_countries']=movies['production_countries'].str.split(', ')

movies.head()

# +
# 处理第二个文件
# -

mov['cast']=mov['cast'].apply(json.loads)
def main_cast(x):
    cnt = 0
    list1 = []
    #只保留主演演员
    for i in x:
        if len(list1)<=10:
            if(i['id'] not in list1):
                list1.append(i['id'])
        else:
            return list1
    return list1
mov['main_cast']=mov['cast'].apply(main_cast)

mov.head()

mov['crew']=mov['crew'].apply(json.loads)
#Crew部分除了director都按department分类
# director
def director(x):
    list1 = []
    for i in x:
        if i['job'] == 'Director':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1
mov['director']=mov['crew'].apply(director)


def editing(x):
    list1 = []
    for i in x:
        if i['department'] == 'Editing':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1
mov['editing']=mov['crew'].apply(editing) 


def sound(x):
    list1 = []
    for i in x:
        if i['department'] == 'Sound':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1
mov['sound']=mov['crew'].apply(sound) 


# +
def art(x):
    list1 = []
    for i in x:
        if i['department'] == 'Art':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1

def production(x):
    list1 = []
    for i in x:
        if i['department'] == 'Production':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1

def visualeffects(x):
    list1 = []
    for i in x:
        if i['department'] == 'Visual Effects':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1

def costume(x):
    list1 = []
    for i in x:
        if i['department'] == 'Costume & Make-Up':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1

def camera(x):
    list1 = []
    for i in x:
        if i['department'] == 'Camera':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1
def crew(x):
    list1 = []
    for i in x:
        if i['department'] == 'Crew':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1

def lighting(x):
    list1 = []
    for i in x:
        if i['department'] == 'Lighting':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1

def writing(x):
    list1 = []
    for i in x:
        if i['department'] == 'Writing':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1

def directing(x):
    list1 = []
    for i in x:
        if i['department'] == 'Directing':
            if(i['id'] not in list1):
                list1.append(i['id'])
    return list1


# -

mov['art']=mov['crew'].apply(art) 
mov['production']=mov['crew'].apply(production) 
mov['visual_effects']=mov['crew'].apply(visualeffects) 
mov['costume']=mov['crew'].apply(costume) 
mov['camera']=mov['crew'].apply(camera)  
mov['lighting']=mov['crew'].apply(lighting) 
mov['writing']=mov['crew'].apply(writing) 
mov['crew_member']=mov['crew'].apply(crew)
mov['directing']=mov['crew'].apply(directing)

mov.head(1)

mov.info()

movies.to_csv('movies_1206id.csv', sep=',', header=True, index=False)
mov.to_csv('mov_1206id.csv', sep = ',', header=True, index=False)
# dataset=movies.merge(mov,left_on='id',right_on='movie_id',how='left')
for col in ['main_cast','director', 'editing', 'sound', 'art', 'production', 
            'visual_effects', 'costume', 'camera', 'lighting', 'writing', 'crew_member']:
    movies[col] = mov[col]
movies.head()

# +
# 加入category
# -

col_name = movies.columns.tolist()
col_name.insert(0,'category')
print(col_name)

movies = movies.reindex(columns=col_name)

movies['category'] = 'Movie'


# +
# 这个算法太慢&维度太高
def uniqueList(col):
    List = []
    for index, row in movies.iterrows():
        col_all = row[col]
        for cell in col_all:
            if cell not in List:
                List.append(cell)
    if List[-1] == '':
        return List[0:-1]
    else:
        return List

def binary(unique_list, cell_list):
    binaryList=[]
    for one in unique_list:
        if one in cell_list:
            binaryList.append(1)
        else:
            binaryList.append(0)
    return binaryList



# -

genresList = uniqueList('genres')

castList = uniqueList('main_cast')

movies['genres_bin'] = movies['genres'].apply(lambda x: binary(genresList,x))

movies['genres_bin'][0]

movies['cast_bin'] = movies['main_cast'].apply(lambda x: binary(castList,x))

movies['cast_bin'].head(3)

directorList = uniqueList('director')

movies['director_bin'] = movies['director'].apply(lambda x: binary(directorList,x))

movies['director_bin'].head()

movies.to_csv('movieNode_1206.csv', sep=',', header=True, index=False)

# +
# dst = pd.read_csv('movieNode_1206.csv')
#存入csv后都会变成字符串，再取出需要重新处理为list
# dst['production_companies'].str.replace('[','').str.replace(']','').str.split(', ')

# +
# 电影-主题表
# -

mov2 = movies['genres_bin']

tmp = [line for line in mov2]
tmp_arr = np.array(tmp)
movie_genres = pd.DataFrame(data = tmp_arr, columns = genresList)

movie_genres.head()

colname = movie_genres.columns.tolist()
colname.insert(0,'movie_id')

movie_genres = movie_genres.reindex(columns=colname)
movie_genres['movie_id'] = movies['id']

movie_genres.head()

movie_genres.to_csv('movie_genres.csv',sep=',', header=True, index=False)

movie_genres.T.head(3)

# +
#电影-主要演员
# -

mov3 = movies['cast_bin']
tmp1 = [line for line in mov3]
tmp_arr1 = np.array(tmp1)
movie_cast = pd.DataFrame(data = tmp_arr1, columns = castList)

colname1 = movie_cast.columns.tolist()
colname1.insert(0,'movie_id')
movie_cast = movie_cast.reindex(columns=colname1)
movie_cast['movie_id'] = movies['id']

movie_cast.to_csv('movie_cast.csv',sep=',', header=True, index=False)

movie_cast.head()

# +
#电影-导演
# -

mov4 = movies['director_bin']
tmp2 = [line for line in mov4]
tmp_arr2 = np.array(tmp2)
movie_director = pd.DataFrame(data = tmp_arr2, columns = directorList)

colname2 = movie_director.columns.tolist()
colname2.insert(0,'movie_id')
movie_director = movie_director.reindex(columns=colname2)
movie_director['movie_id'] = movies['id']

movie_director.head()

movie_director.to_csv('movie_director.csv',sep=',', header=True, index=False)

# +
# Cast_node
# -

mov['cast'][0][0]

castNode = pd.DataFrame(data = mov['cast'][0])
colname3 = castNode.columns.tolist()
colname3.insert(7,'movie_id')
castNode = castNode.reindex(columns=colname3)
castNode['movie_id'] = movies['id'][0]

for i in range(1,4803):
    tempNode = pd.DataFrame(data = mov['cast'][i])
    tempNode = tempNode.reindex(columns=colname3)
    tempNode['movie_id'] = movies['id'][i]
    castNode = castNode.append(tempNode)
    print("i=",i)

castNode[['cast_id','gender','id','order']] = castNode[['cast_id','gender','id','order']].astype(int) 

castNode['category'] = 'Cast'

castNode.to_csv('castNode.csv',sep=',', header=True, index=False)

castNode

#简单、合并版castNode
castNode1 = castNode.drop(['cast_id','character','credit_id','order'],axis=1,inplace=False)

castNode1.head()

castNode1[['movie_id']] = castNode1[['movie_id']].astype(str) 


# +
# 按name合并多行

def ab(df):
    return','.join(df.values)

def simplify(x):
    line = x.split(',')
    return ','.join(list(set(line)))


# -

castNode1 = castNode1.groupby(['id','name','gender'])['movie_id'].apply(ab)
castNode1 = castNode1.reset_index()

castNode1['movie_id'] = castNode1['movie_id'].apply(lambda x: simplify(x))

castNode1['category'] = 'Cast' 

castNode1.head(2)

castNode1.to_csv('castNode_simplified.csv',sep=',', header=True, index=False)

# +
# CrewNode
# -

mov['crew'][0][0]

crewNode = pd.DataFrame(data = mov['crew'][0])
colname4 = crewNode.columns.tolist()
colname4.insert(7,'movie_id')
crewNode = crewNode.reindex(columns=colname4)
crewNode['movie_id'] = movies['id'][0]

for i in range(1,4803):
    tempNode1 = pd.DataFrame(data = mov['crew'][i])
    tempNode1 = tempNode1.reindex(columns=colname4)
    tempNode1['movie_id'] = movies['id'][i]
    crewNode = crewNode.append(tempNode1)
    print("i=",i)

crewNode[['gender','id']] = crewNode[['gender','id']].astype(int) 

crewNode1 = crewNode.drop(['credit_id'],axis=1,inplace=True)

crewNode['category'] = 'Crew'

crewNode.to_csv('crewNode.csv',sep=',', header=True, index=False)



crewNode.head()

# +
# Director Node
# -

directorNode = crewNode[crewNode['job']=='Director']
# directorNode1 = directorNode.drop(['credit_id'],axis=1,inplace=True)

directorNode.head()

directorNode = directorNode.reset_index()
directorNode2 = directorNode.drop(['index'],axis=1,inplace=True)

directorNode[['movie_id']] = directorNode[['movie_id']].astype(str) 

directorNode['category'] = 'Director'


# +
# 按name合并多行
def ab(df):
    return','.join(df.values)
    
directorNode = directorNode.groupby(['id','name','gender'])['movie_id'].apply(ab)
directorNode = directorNode.reset_index()
# -

directorNode['movie_id'][0]

directorNode.head()

directorNode.to_csv('directorNode.csv',sep=',', header=True, index=False)

#Movie Node最后的简化
movies.head(1)

movies.info()

movies1 = movies.drop(['homepage','spoken_languages','status'],axis=1,inplace=False)

movies1['category'] = 'Movie'

movies1.to_csv('movieNode_1206.csv',sep=',', header=True, index=False)

movies1.info()

movies1.head(1)



StarcoderdataJupyterScriptsDedupFiltered