# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.8.6 64-bit
#     name: python3
# ---

# # Data Preparation
#
# ### In this notebook we will prepare our data for our search function to use. <br>Currently we have data stored in four different ```csv``` files.<br>
# * movies.csv
# * links.csv
# * ratings.csv
# * tags.csv
# <br>
# ### It can be computationally expensive to produce ```analysis results``` from multiple data-sources for incomming stream of requests.<br> So we will prepare our data and save it in an ```easily searchable``` structure.

# Import the needed modules...
import pandas as pd
from collections import defaultdict
from os import getcwd

# ## Define Paths to data files.

PATH_LINKS   = f"{getcwd()}/dataStore/links.csv"
PATH_MOVIES  = f"{getcwd()}/dataStore/movies.csv"
PATH_RATINGS = f"{getcwd()}/dataStore/ratings.csv"
PATH_TAGS    = f"{getcwd()}/dataStore/tags.csv"

# # Data Engineering<br>
# * ## Get data in dataframes.
# * ## Convert data to a single dictionary.

"""
    Read data from movies.csv
"""
df_movies            = pd.read_csv(PATH_MOVIES)
movies_table_columns = df_movies.columns.tolist()
print(f"COLUMNS : {movies_table_columns}")

"""
    Read data from links.csv
"""
df_links            = pd.read_csv(PATH_LINKS)
links_table_columns = df_links.columns.tolist()
print(f"COLUMNS : {links_table_columns}")

"""
    Read data from ratings.csv
"""
df_ratings         = pd.read_csv(PATH_RATINGS)
path_table_columns = df_ratings.columns.tolist()
print(f"COLUMNS : {path_table_columns}")

"""
    Read data from tags.csv
"""
df_tags            = pd.read_csv(PATH_TAGS)
tags_table_columns = df_tags.columns.tolist()
print(f"COLUMNS : {tags_table_columns}")

# * ### ```movieId``` is a common column in all four tables so we will use it as a primary search-key <br>
# * ### ```userId```  is a common key across two tables, so we will use it as a sort key...
# * ### A user will always search a movie by its ```title``` so we will create a ```Global secondary index``` to be able to perform search our datastore. <br>it will obviously take some extra space but almost negligible as compared to the size of the original data. <br>In addition, It will make our searching faster and efficient so it's a good deal.

# +
print(f"It is {pd.Series(df_movies['movieId']).is_unique}  that the column 'movieId' has unique values for all entries in movies dataframe.")
print(f"It is {pd.Series(df_links['movieId']).is_unique}  that the column 'movieId' has unique values for all entries in links dataframe.")
print(f"It is {pd.Series(df_ratings['userId']).is_unique} that the column 'userId'  has unique values for all entries in ratings dataframe.")
print(f"It is {pd.Series(df_tags['userId']).is_unique} that the column 'userId'  has unique values for all entries in tags dataframe.")

# Sort movie dataframe on the basis of movieId as movieId is unique for all entries...
df_movies_sorted = df_movies.sort_values(by=['movieId'])

# Sort links dataframe on the basis of movieId as movieId is unique for all entries...
df_links_sorted  = df_links.sort_values(by=['movieId'])

# +
# from movies dataframe...
movieIds    = df_movies_sorted["movieId"].tolist()
movieTitles = df_movies_sorted["title"].tolist()
movieGenres = [genre.split("|") for genre in df_movies["genres"].tolist()]

# from links dataframe...
imdbId  = df_links_sorted["imdbId"].tolist()
tmdbId  = df_links_sorted["tmdbId"].tolist()
# -

movieDict             = {}
global_secondaryIndex = {}
for idx, movieId in enumerate(movieIds):
    movieDict[movieId] = {
        "genre" : movieGenres[idx],
        "links" : {
            "imdb" : imdbId[idx], 
            "tmdb" : tmdbId[idx]
        }
    }
    
    global_secondaryIndex[movieTitles[idx]] = movieId

# delete veriables which are no longer in use while holding large amount of data.
del movieIds
del movieTitles
del movieGenres
del imdbId
del tmdbId

# ### Add all user ratings for individual movies.
# #### The goal is to group all ratings of a ```movie``` togather, so that we will be able to retrieve user ratings of a particular movie.
# #### Now, this one is a bit tricky as there is no column in the ratings dataframe which offers unique values. <br>So will have to perform grouping.
# #### We will use ```movieId``` column as it is a common column in all of our data sources and it will make it easy to add the same data in our  new ```movie``` dataset.
#
# #### The procedure defined below may be computationally gross but should be good enough for a single time execution...

# +
# convert all columns of ratings table into individual lists...
userIds       = df_ratings["userId"].tolist()
movieIds      = df_ratings["movieId"].tolist()
user_ratings  = df_ratings["rating"].tolist()
timestamps    = df_ratings["timestamp"].tolist()

ratings = {}

for idx, mid in enumerate(movieIds):
    # Do the movieId previously exist?
    try   : _ = ratings[mid]
    # If not, Add it in the record...
    except: 
        ratings[mid]   = [
            {
                "userId"     : userIds[idx],
                "rating"     : user_ratings[idx],
                "time_stamp" : timestamps[idx]
            }
        ]
    
    try   : _ = ratings[mid][userIds[idx]]
    except: ratings[mid].append(
            {
                "userId"     : userIds[idx],
                "rating"     : user_ratings[idx],
                "time_stamp" : timestamps[idx]
            }
        )

# Finally, add the data in the movieDict...
for mid, _ in movieDict.items():
    try   : movieDict[mid]["user_rating"] = ratings[mid][1:]
    except: 
        try   : movieDict[mid]["user_rating"] = [] # If Movie ID exists in the movie dict...
        except: pass # If the Movie ID doesn't exist in our record...
# -

# ### Add all user given tags for individual movies.
# #### The goal is to group all tags given to a ```movie``` togather, so that we will be able to retrieve tags of a particular movie.
# #### This one is also tricky as there is no column in the tags dataframe which offers unique values. <br>So will have to perform grouping.
# #### We will use ```movieId``` column as it is a common column in all of our data sources and it will make it easy to add the same data in our  new ```movie``` dataset.
#
# #### The procedure defined below may also be computationally gross but should be good enough for a single time execution...

# +
# convert all columns of ratings table into individual lists...
userIds    = df_tags["userId"].tolist()
movieIds   = df_tags["movieId"].tolist()
user_tag   = df_tags["tag"].tolist()
timestamps = df_tags["timestamp"].tolist()

tags = {}
for idx, mid in enumerate(movieIds):
    # Do the movieId previously exist?
    try   : _ = tags[mid]
    # If not, Add it in the record...
    except: tags[mid] = [
        {
            "userId"     : userIds[idx],
            "rating"     : user_tag[idx],
            "time_stamp" : timestamps[idx]
        }
    ]
    
    try   : _ = ratings[mid][userIds[idx]]
    except: tags[mid].append(
            {
                "userId"     : userIds[idx],
                "rating"     : user_tag[idx],
                "time_stamp" : timestamps[idx]
            }
        )

# Finally, add the data in the movieDict...
for mid, _ in movieDict.items():
    try   : movieDict[mid]["tags"] = tags[mid][1:]
    except: 
        try   : movieDict[mid]["tags"] = [] # If Movie ID exists in the movie dict...
        except: del global_secondaryIndex[mid]
# -

import json
print("[INFO] Writing movie Data into the disk...")
with open('dataStore/dataFinal.json', 'w') as fp:
    json.dump(movieDict, fp, sort_keys=True, indent=4)
print("[INFO] Writing Global Secondary Index Data into the disk...")
with open('dataStore/dataFinal_GIS.json', 'w') as fp:
    json.dump(global_secondaryIndex, fp, sort_keys=True, indent=4)

# #### At this point, our database is ready and it can handel high inflow of requests.

StarcoderdataJupyterScriptsDedupFiltered