# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="giR0ibyi9GZm"
# # **Set Up**

# + id="DZEFKIjKyMxw" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1617673674143, "user_tz": 240, "elapsed": 382, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="261fd6db-7ab1-4a96-f80e-3c2db32537b3"
# import libraries
import pandas as pd
import numpy as np

# %matplotlib inline
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import pickle
import os

import datetime
from wordcloud import WordCloud

from tensorflow import keras 
import tensorflow as tf
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint 
from keras.models import Model
from keras.layers import Input, Reshape, Dot
from keras.layers.embeddings import Embedding

print("Keras version " + keras.__version__)
print("Tensorflow version " + tf.__version__)

# + id="8yw6-4Tq2kBX" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1617673521442, "user_tz": 240, "elapsed": 20599, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="1ebff91c-e5c1-47be-afaf-86b5ff671e77"
from google.colab import drive
drive.mount("/content/drive")

# + [markdown] id="QPzFFR3i9iV-"
# # **Data Loading and Preprocessing**

# + id="nzDU60oQ2pZG" executionInfo={"status": "ok", "timestamp": 1617673523197, "user_tz": 240, "elapsed": 354, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
base_path = '/content/drive/My Drive/DATA2040_Final_Project_YARD'

# + id="ksMBFb5UV7NG" executionInfo={"status": "ok", "timestamp": 1617673526275, "user_tz": 240, "elapsed": 2609, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
ratings = pd.read_csv(base_path + '/data/ratings.csv', sep='\t', encoding='latin-1', 
                      usecols=['user_id', 'movie_id', 'user_emb_id', 'movie_emb_id', 'rating'])

# + colab={"base_uri": "https://localhost:8080/", "height": 534} id="Ye80pMPCWFZm" executionInfo={"status": "ok", "timestamp": 1617673527382, "user_tz": 240, "elapsed": 305, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="aea6b170-d76e-43a6-ba36-b501c7085c4c"
ratings

# + colab={"base_uri": "https://localhost:8080/"} id="BURhMhv9WMUw" executionInfo={"status": "ok", "timestamp": 1617673529177, "user_tz": 240, "elapsed": 283, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="ed9e60cb-6ef5-425e-8021-bf26c13df07a"
max_userid = ratings['user_id'].drop_duplicates().max()
max_movieid = ratings['movie_id'].drop_duplicates().max()
print(max_userid)
print(max_movieid)

# + id="5e52Tozo2JGL" executionInfo={"status": "ok", "timestamp": 1617673530862, "user_tz": 240, "elapsed": 628, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
users = pd.read_csv(base_path + '/data/users.csv', sep='\t', encoding='latin-1', 
                    usecols=['user_id', 'gender', 'zipcode', 'age_desc', 'occ_desc'])

# + colab={"base_uri": "https://localhost:8080/", "height": 534} id="SCvEQn4m3H4_" executionInfo={"status": "ok", "timestamp": 1617673531877, "user_tz": 240, "elapsed": 243, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="29059ccc-1195-4c64-fdce-3980ae613fb5"
users

# + id="kH9Vvx5L6KTi" executionInfo={"status": "ok", "timestamp": 1617673534013, "user_tz": 240, "elapsed": 408, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
movies = pd.read_csv(base_path + '/data/movies.csv', sep='\t', encoding='latin-1', 
                     usecols=['movie_id', 'title', 'genres'])

# + colab={"base_uri": "https://localhost:8080/", "height": 534} id="utCVhoGyWcrc" executionInfo={"status": "ok", "timestamp": 1617673534929, "user_tz": 240, "elapsed": 399, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="b205dd00-948f-43ef-c3fa-2301348aed31"
movies

# + colab={"base_uri": "https://localhost:8080/"} id="UUJYGu9yWuNw" executionInfo={"status": "ok", "timestamp": 1617673536813, "user_tz": 240, "elapsed": 403, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="9d47b459-9915-4d93-9ce8-d54c1c667621"
# Create training set
RNG_SEED = 32
shuffled_ratings = ratings.sample(frac=1., random_state=RNG_SEED)

# Shuffling users
Users = shuffled_ratings['user_emb_id'].values
print('Users:', Users, ', shape =', Users.shape)

# Shuffling movies
Movies = shuffled_ratings['movie_emb_id'].values
print('Movies:', Movies, ', shape =', Movies.shape)

# Shuffling ratings
Ratings = shuffled_ratings['rating'].values
print('Ratings:', Ratings, ', shape =', Ratings.shape)

# + [markdown] id="KIJ2nLRr9rZk"
# # **Exploratory Data Analysis**

# + [markdown] id="IwCV-9wiGYXh"
# **MOVIES**

# + colab={"base_uri": "https://localhost:8080/"} id="1oeAdHUhm_RK" executionInfo={"status": "ok", "timestamp": 1617673539515, "user_tz": 240, "elapsed": 265, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="17c51f2e-ce72-4298-c712-7d4b32fdf55f"
# get movie shape
movies.shape

# + [markdown] id="I8tA_IT7pV5Z"
# There are 3883 movies in the dataset. The first column contains the movie ID (integers), the second column contains the movie titles (along with the year it was released), and the third column contains the movie genres. 

# + colab={"base_uri": "https://localhost:8080/", "height": 260} id="QfQALp1dorMQ" executionInfo={"status": "ok", "timestamp": 1617673541385, "user_tz": 240, "elapsed": 245, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="10096b2d-da74-4a0c-98b7-eb6c4ffdb537"
movies.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 380} id="4Gih5TaSn0Ji" executionInfo={"status": "ok", "timestamp": 1617673542816, "user_tz": 240, "elapsed": 281, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="29aac919-299f-4deb-be24-502fad230f1b"
movies.describe()

# + colab={"base_uri": "https://localhost:8080/"} id="AWyO92OEqjP5" executionInfo={"status": "ok", "timestamp": 1617673544163, "user_tz": 240, "elapsed": 235, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="3e238acc-d787-4517-95b1-b312be97424f"
# check for missing values
movies.isnull().sum(axis=0)

# + colab={"base_uri": "https://localhost:8080/"} id="rCd4sxkOqrl3" executionInfo={"status": "ok", "timestamp": 1617673545791, "user_tz": 240, "elapsed": 231, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="a6b2a4ea-54b5-4483-ab52-f23a2f288137"
# check count of different genres
movies['genres'].value_counts()

# + colab={"base_uri": "https://localhost:8080/"} id="dugRcolCq20A" executionInfo={"status": "ok", "timestamp": 1617673547197, "user_tz": 240, "elapsed": 311, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="5d83165c-c2ca-4b9a-9c4d-0ec51ae4099c"
# check number of unique genres
movies['genres'].value_counts().nunique()

# + id="0U5FfV9Xq7yY" executionInfo={"status": "ok", "timestamp": 1617673548831, "user_tz": 240, "elapsed": 428, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
# merge movies and ratings
movies_ratings = pd.merge(movies, ratings)

# + colab={"base_uri": "https://localhost:8080/", "height": 260} id="Abnm3pllriiZ" executionInfo={"status": "ok", "timestamp": 1617673550458, "user_tz": 240, "elapsed": 312, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="612c925d-4274-4e58-9509-63493bb46a4d"
movies_ratings.head()

# + colab={"base_uri": "https://localhost:8080/"} id="ExzwYdYEryFu" executionInfo={"status": "ok", "timestamp": 1617673551825, "user_tz": 240, "elapsed": 255, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="730092b2-282a-4e3c-f348-3e1e7e700bce"
top10 = movies_ratings.groupby('title').size().sort_values(ascending=False)[:10]
top10

# + colab={"base_uri": "https://localhost:8080/", "height": 590} id="DJHlHpy7VhS_" executionInfo={"status": "ok", "timestamp": 1617673554649, "user_tz": 240, "elapsed": 1555, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="775c5ae7-f006-48c9-e50f-0ce2f4768c03"
#Plot Top 10 Movies Viewed
movies_ratings.groupby('title')['user_id'].count().sort_values(ascending = False)[:10].plot(kind ='barh', figsize = (10,9), color= "pink")
plt.xlabel('Count')
plt.ylabel('Movie Titles')
plt.title('Top 10 Viewed Movies')
plt.savefig(base_path + '/figures/plt/' + 'Bar chart Top 10 Movies Viewed.jpg',bbox_inches='tight')
plt.show

# + [markdown] id="uwNfcj0gWD_r"
# The graph shows the top 10 movies viewed (each user can only be counted once) by count. American Beauty is the highest followed by the 3 Star Wars Movies.

# + colab={"base_uri": "https://localhost:8080/"} id="8pCMRFCiTQVL" executionInfo={"status": "ok", "timestamp": 1617673556363, "user_tz": 240, "elapsed": 292, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="391d2f5d-6b74-489b-8671-3818f32d3e21"
#counts unique genres
n = len(pd.unique(movies_ratings ['genres']))
print("No.of.unique values :", n)


# + id="orOaxQfrTmlN" executionInfo={"status": "ok", "timestamp": 1617673557759, "user_tz": 240, "elapsed": 252, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
#define a function that counts the number of times each genre appear:
def count_word(df, ref_col, liste):
    keyword_count = dict()
    for s in liste: keyword_count[s] = 0
    for liste_keywords in movies_ratings[ref_col].str.split('|'):
        if type(liste_keywords) == float and pd.isnull(liste_keywords): continue
        for s in liste_keywords: 
            if pd.notnull(s): keyword_count[s] += 1
    # convert the dictionary in a list to sort the keywords  by frequency
    keyword_occurences = []
    for k,v in keyword_count.items():
        keyword_occurences.append([k,v])
    keyword_occurences.sort(key = lambda x:x[1], reverse = True)
    return keyword_occurences, keyword_count


# + id="ny8bBZaiTzkh" executionInfo={"status": "ok", "timestamp": 1617673561985, "user_tz": 240, "elapsed": 2975, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
genre_labels = set()
for s in movies_ratings['genres'].str.split('|').values:
    genre_labels = genre_labels.union(set(s))

# + colab={"base_uri": "https://localhost:8080/"} id="zXFrgj5wTm2p" executionInfo={"status": "ok", "timestamp": 1617673566448, "user_tz": 240, "elapsed": 3785, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="26aee761-d691-4eb9-d6be-2641404c4e01"
keyword_occurences, dum = count_word(movies_ratings, 'genres', genre_labels)
#keyword_occurences
trunc_occurences = keyword_occurences[0:50]
trunc_occurences

# + colab={"base_uri": "https://localhost:8080/", "height": 492} id="slA4jDJtUOgO" executionInfo={"status": "ok", "timestamp": 1617673568614, "user_tz": 240, "elapsed": 1116, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="4f86abbc-f784-4d2f-83fd-239982c5b364"
# Bar graph of popularity of genres
fig = plt.figure(1, figsize=(18,13))
ax2 = fig.add_subplot(2,1,2)
y_axis = [i[1] for i in trunc_occurences]
x_axis = [k for k,i in enumerate(trunc_occurences)]
x_label = [i[0] for i in trunc_occurences]
plt.xticks(rotation=85, fontsize = 15)
plt.yticks(fontsize = 15)
plt.xticks(x_axis, x_label)
plt.ylabel("Number of Occurences", fontsize = 24, labelpad = 0)
ax2.bar(x_axis, y_axis, align = 'center', color = 'purple')
plt.title("Popularity of Genres",fontsize = 30)
plt.savefig(base_path + '/figures/plt/' + 'Bar graph popularity of genres.jpg',bbox_inches='tight')
plt.show()


# + [markdown] id="FzJMeCRfVLfj"
# The bar graph shows the number of occurences for each genre(genre was edited because there were too many genres). Some movies had multiple genres (ex. romantic comedy woud count as romance and comedy) We can see that comedy and drama are the most present genres. 

# + colab={"base_uri": "https://localhost:8080/", "height": 358} id="mZXagWG-V8Zt" executionInfo={"status": "ok", "timestamp": 1617673679907, "user_tz": 240, "elapsed": 1085, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="4c34b77d-d566-4d6a-84c7-088b8ed71c31"
#Wordcloud visualization of popularity of genres
# Function that control the color of the words
def random_color_func(word=None, font_size=None, position=None,
                      orientation=None, font_path=None, random_state=None):
    h = int(360.0 * tone / 255.0)
    s = int(100.0 * 255.0 / 255.0)
    l = int(100.0 * float(random_state.randint(70, 120)) / 255.0)
    return "hsl({}, {}%, {}%)".format(h, s, l)


#Finally, the result is shown as a wordcloud:
words = dict()
trunc_occurences = keyword_occurences[0:50]
for s in trunc_occurences:
    words[s[0]] = s[1]
tone = 100 # define the color of the words
f, ax = plt.subplots(figsize=(14, 6))
wordcloud = WordCloud(width=550,height=300, background_color='black', 
                      max_words=1628,relative_scaling=0.7,
                      color_func = random_color_func,
                      normalize_plurals=False)
wordcloud.generate_from_frequencies(words)
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis('off')
plt.savefig(base_path + '/figures/plt/' + 'Wordcloud visualization of popularity of genres.jpg',bbox_inches='tight')
plt.show()

# + [markdown] id="f33zIy0VWlmg"
# This Wordcloud visualization of popularity of genres shows the same things as the bar grah above which is the number of occurences for each genre(genre was edited because there were too many genres). Some movies had multiple genres (ex. romantic comedy woud count as romance and comedy) Again, we can see that comedy and drama are the most present genres.

# + [markdown] id="jh9dMS4pC6WD"
# **RATINGS**

# + colab={"base_uri": "https://localhost:8080/", "height": 380} id="4kJSeb4TtIpR" executionInfo={"status": "ok", "timestamp": 1617673581795, "user_tz": 240, "elapsed": 425, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="594a59f5-2a05-4b0f-c306-a81548410407"
ratings.describe()

# + colab={"base_uri": "https://localhost:8080/", "height": 306} id="iUpwjyuAtXcB" executionInfo={"status": "ok", "timestamp": 1617673584194, "user_tz": 240, "elapsed": 806, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="9cca842a-f94c-47a0-f28c-d0eb20011ac5"
# plot a bar chart of ratings count
ratings.value_counts(ratings['rating']).sort_index().plot.bar(color=["orange"])
plt.title('Distribution of Movie Ratings', fontsize=20)
plt.ylabel('count', fontsize=16)
plt.xlabel(ratings['rating'].name, fontsize=16)
plt.xticks(rotation=0, ha='right')
plt.savefig(base_path + '/figures/plt/' + 'Bar chart ratings count.jpg',bbox_inches='tight')
plt.show()

# + colab={"base_uri": "https://localhost:8080/", "height": 306} id="HdMD_7y1Tm9n" executionInfo={"status": "ok", "timestamp": 1617673585755, "user_tz": 240, "elapsed": 561, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="af9ecfa8-936f-4dc7-d52a-1b519226f8d4"
# plot a bar chart of ratings count
ratings.value_counts(ratings['rating'], normalize=True).sort_index().plot.bar(color=["yellow"])
plt.title('Distribution of Movie Ratings', fontsize=20)
plt.ylabel('proportion', fontsize=16)
plt.xlabel(ratings['rating'].name, fontsize=16)
plt.xticks(rotation=0, ha='right')
plt.savefig(base_path + '/figures/plt/' + 'Bar chart ratings proportion.jpg',bbox_inches='tight')
plt.show()

# + colab={"base_uri": "https://localhost:8080/"} id="jZdMzOtdUK7F" executionInfo={"status": "ok", "timestamp": 1617673587359, "user_tz": 240, "elapsed": 200, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="d4d22f81-d0b8-47c2-9329-9e2cf24e17fe"
# get exact class proportions
ratings.value_counts(ratings['rating'], normalize=True)

# + colab={"base_uri": "https://localhost:8080/"} id="glUzMlCoInZj" executionInfo={"status": "ok", "timestamp": 1617673588829, "user_tz": 240, "elapsed": 229, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="4ec8f208-31c1-4f1d-dd0c-438733fde512"
# mean ratings
np.mean(ratings['rating'])

# + [markdown] id="DCFa8mGoGxSB"
# The bar plot shows the distribution of movie ratings. The mode (i.e. most common) rate is 4, and the distribution is left-skewed. The mean rating is 3.58.
#
# In terms of class balance, 34.9% of the ratings are 4s, 26.1% are 3s, 22.6% are 5s, 10.7% are 2s, and 5.6% are 1s.

# + colab={"base_uri": "https://localhost:8080/", "height": 480} id="8wlNtO5xNmPP" executionInfo={"status": "ok", "timestamp": 1617673590758, "user_tz": 240, "elapsed": 861, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="8624adb8-b3a8-459e-b62a-888fc3ed8cba"
#Visualize the User Rating of the Movie “Toy Story”
movies_ratings[movies_ratings.title == 'Toy Story (1995)'].groupby('rating')['user_id'].count().plot(kind = 'bar', color = 'green',figsize = (8,7))
plt.xlabel('Rating')
plt.ylabel('Number of Users')
plt.title('Bar Chart of User Rating of Toy Story')
plt.xticks(rotation=0, ha='right')
plt.savefig(base_path + '/figures/plt/' + 'Bar chart User Rating of the Movie “Toy Story”.jpg',bbox_inches='tight')
plt.show

# + [markdown] id="rT_MkJ7IONVM"
# The bar plot shows the distribution of movie ratings for Toy Story by user. Clearly, we can see that Toy Story has received very good reviews give that it has mostly 4 and 5 stars

# + [markdown] id="DBruhJpHDDPr"
# **USERS**

# + [markdown] id="8LhFWX_NDuuy"
# **Gender**

# + colab={"base_uri": "https://localhost:8080/"} id="x55az-WzvZwk" executionInfo={"status": "ok", "timestamp": 1617673592754, "user_tz": 240, "elapsed": 230, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="e3403a01-9f84-48d2-d0d3-61ee3655f362"
# get gender count
users['gender'].value_counts()

# + colab={"base_uri": "https://localhost:8080/", "height": 248} id="dBL2GW_xybUw" executionInfo={"status": "ok", "timestamp": 1617673594039, "user_tz": 240, "elapsed": 603, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="fa66f138-c6a6-4e5d-b76f-c86eddfdc3cc"
# plot a pie chart of gender distribution
gender_data = users.value_counts(users['gender'])
plt.pie(gender_data, labels = ["male", "female"], autopct='%.2f', colors=["lightblue", "pink"], textprops={'fontsize': 14})
plt.savefig(base_path + '/figures/plt/' + 'Pie chart of gender distribution.jpg',bbox_inches='tight')
plt.show()

# + [markdown] id="cD-sagc_GItG"
# From the pie chart, 71.7% of the users are male, and 28.3% are female. Thus, the data is imbalanced towards male users.

# + [markdown] id="fASSkNASD2Ax"
# **Age range**

# + colab={"base_uri": "https://localhost:8080/", "height": 284} id="Ho7qHWph0ern" executionInfo={"status": "ok", "timestamp": 1617673596105, "user_tz": 240, "elapsed": 975, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="d3826580-48f1-48e0-b57c-5ceeb6c2e5f8"
# replace below 18 values with 0-18, for plot order
users['age_desc'] = users['age_desc'].replace('Under 18', '0-18')
users.head()

# plot a bar chart of user age ranges
users.value_counts(users['age_desc']).sort_index().plot.bar(color=["orange"])
plt.ylabel('count', fontsize=16)
plt.xlabel('user age range', fontsize=16)
plt.xticks(rotation=0, ha='right')
plt.savefig(base_path + '/figures/plt/' + 'Bar chart of user age ranges.jpg',bbox_inches='tight')
plt.show()

# + [markdown] id="0aVo8H9-E902"
# From the bar plot of user age ranges, the age distribution has its mode at 25-34, and appears to have a right skew. Overall, most users appear to be young adults between 18-44.

# + colab={"base_uri": "https://localhost:8080/"} id="p-nqnidQ8PCo" executionInfo={"status": "ok", "timestamp": 1617673597360, "user_tz": 240, "elapsed": 237, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}} outputId="82b68d0d-c288-4182-8d2f-1bc6353f6583"
users.value_counts(users['age_desc'])

# + id="FGvdO3Uv0lVM" executionInfo={"status": "ok", "timestamp": 1617673599296, "user_tz": 240, "elapsed": 253, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiYl_tqOFZ5W2cE-YU8KG1J1TwyYbIywmPedKYmNQ=s64", "userId": "14394843052745211035"}}
age_ranges = users['age_desc'].unique()

# + [markdown] id="ZJdHIwFaD_oJ"
# **Occupation**

# + colab={"base_uri": "https://localhost:8080/", "height": 284} id="PD7bwacf1hs9" executionInfo={"status": "ok", "timestamp": 1617673601139, "user_tz": 240, "elapsed": 981, "user": {"displayName": "<NAME>", "photoUrl": "<KEY>", "userId": "14394843052745211035"}} outputId="5f86dd5c-f0f8-42df-c850-41a39a17061a"
# get list of occupations
users.value_counts(users['occ_desc']).nunique()

# plot a sorted horizontal bar chart of user occupations
users.value_counts(users['occ_desc']).sort_values(ascending=True).plot.barh(color=["orange"])
plt.ylabel('count', fontsize=16)
plt.xlabel('user occupations', fontsize=16)
plt.xticks(rotation=0, ha='right')
plt.savefig(base_path + '/figures/plt/' + 'Sorted horizontal bar chart of user occupations.jpg',bbox_inches='tight')
plt.show()

# + [markdown] id="AQpOLXqu96DD"
# There are 21 different occupations in the dataset. The most common occupation in the Movie Lens user data is college/grad student, followed by 'other, and then 'executive managerial'. The top 5 most common occuaptions also include academic/educator, and technician engineering. Thus, users appear to be relatively highly educated.

StarcoderdataJupyterScriptsDedupFiltered