# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="lE68BuQxQ772"
# # Knowledge Based Recommendation System of Ingredients
# ## Notebook 1: Exploratory Data Analysis and Preprocessing
# ### Project Breakdown
#     1  Exploratory Data Analysis and Preprocessing
#     2: Build Word Embeddings using Word2Vec, FastText
#     3: Recommend Recipes based on ingredients
#     4: Build and Visualize Interactive Knowledge Graph of Ingredients
#
#
#
# [This is the dataset](https://eightportions.com/datasets/Recipes/#fn:1) we will be using. It is collated by <NAME>, sourced from [Food Network](https://www.foodnetwork.com/), [Epicurious](https://www.epicurious.com/), and [Allrecipes](https://www.allrecipes.com/).

# + [markdown] id="S-9PrAvfYtTA"
# ## Data Preparation

# + id="zw5GYmrfQ77-"
from nltk.stem.wordnet import WordNetLemmatizer
from wordcloud import WordCloud
from bs4 import BeautifulSoup
from gensim.parsing.preprocessing import remove_stopwords
from tqdm import tqdm
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import pickle
import string
import nltk
import json
import re

# + colab={"base_uri": "https://localhost:8080/"} id="Gs19jnUiSVwi" outputId="2e4ca4b7-d018-4db7-a840-7819965c4e19"
# download recipe data
# !mkdir -p data
# !wget --header="Host: storage.googleapis.com" --header="User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" --header="Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" --header="Accept-Language: en-US,en;q=0.9" --header="Referer: https://eightportions.com/" "https://storage.googleapis.com/recipe-box/recipes_raw.zip" -c -O 'data/recipes_raw.zip'

# + colab={"base_uri": "https://localhost:8080/"} id="SL3YaFfxScGT" outputId="74d2822b-27a9-487e-a42c-eb3cd8985d2c"
# !unzip -xo data/recipes_raw.zip -d data

# + id="O2iHPysjQ78A"
# we scrapped data from three sources
recipe_sources = ['ar', 'epi', 'fn']

# + id="38_lYOxlQ78B"
# We will iterate all json files to build a dataframe
recipes = pd.DataFrame()

sources, titles, ingredients, instructions = [], [], [], []

for recipe_source in recipe_sources:
    with open(f'data/recipes_raw_nosource_{recipe_source}.json', 'r') as fn:
        data = json.load(fn)
        for _, recipe in data.items():
            if ('title' in recipe) and ('ingredients' in recipe) and ('instructions' in recipe):
                # append to a list of the source
                sources.append(recipe_source)

                # append to a list of the titles
                titles.append(recipe['title'])

                # append to a list of a list of ingredients, removing the word ADVERTISEMENT
                # Eg: {'ingredients': ['1 cup butter, softened ADVERTISEMENT', ..]}
                ingredients.append(
                    [str(ingredient).replace('ADVERTISEMENT', '') for ingredient in recipe['ingredients']]
                )

                # append to a list of instructions, remmoving the word ADVERTISEMENT and replace \n with space characters
                instructions.append(
                    str(recipe['instructions']).replace('ADVERTISEMENT', '')
                )

recipes['source'] = sources
recipes['title'] = titles
recipes['ingredients'] = ingredients
recipes['instructions'] = instructions

# + colab={"base_uri": "https://localhost:8080/", "height": 424} id="D4mbBBTkX063" outputId="b95214e6-27d2-4dfe-e8a5-d31658c1eead"
recipes


# + id="qFSIeA4A4qlF"
def plot_wordcloud(data, title):

    wc = WordCloud(max_words=100, max_font_size=50, background_color='white')

    #generate wordcloud from word_freq
    wc = wc.generate(data.str.cat(sep=' '))

    #plot wordcloud
    plt.figure(figsize=(12, 5))
    plt.imshow(wc, interpolation='bilinear')
    plt.title(f'WordCloud of Recipe  {title}', fontsize=22)
    plt.axis('off')
    plt.show()


# + colab={"base_uri": "https://localhost:8080/", "height": 325} id="hR8ohfpN5d8C" outputId="e6d0be2a-d82a-4769-dc77-e4d26a4ba3c5"
plot_wordcloud(recipes['title'], 'Titles')

# + [markdown] id="npvBOPj-6XjQ"
# Most recipes are about dishes related to Chicken, Orange, Shrimp, Cake.

# + colab={"base_uri": "https://localhost:8080/", "height": 325} id="NJ7EQoos5137" outputId="981dc0cb-0214-4364-ac67-2a8caa77d841"
plot_wordcloud(recipes['instructions'], 'Instructions')

# + [markdown] id="DTmjrHMk6k3k"
# Preheat oven is the most appearing instruction.

# + colab={"base_uri": "https://localhost:8080/", "height": 325} id="uPKTx8wP6SMg" outputId="a660265c-30ed-49f4-87ee-ba77f445cc55"
plot_wordcloud(recipes['ingredients'].apply(lambda x: ' '.join(x)), 'Ingredients')

# + [markdown] id="GDqy_zYI7IfG"
# Most preferred ingredient in most of the recipes are seasonings like black pepper, olive oil, and flour

# + [markdown] id="4F7JB1NhYpbn"
# ## Preprocessing and Data Preparation

# + id="UyC0l33IQ78C"
# We will consider following commonly used words as they won't be useful for our analysis
# source: https://en.wikibooks.org/wiki/Cookbook:Units_of_measurement

to_remove = [
    'tablespoon',
    'tablespoons',
    'teaspoon',
    'teaspoons',
    'tsp',
    'tsps',
    'tbsp',
    'tbsps',
    'pound',
    'pounds',
    'grams',
    'mg',
    'mm',
    'm',
    'ounce'
    'ounces',
    'kg',
    'inch',
    'crushed',
    'chopped',
    'finely',
    'softened',
    'cups',
    'cup'
]

# decontract words
# https://stackoverflow.com/a/47091490/4084039

def decontracted(phrase):
    # specific
    phrase = re.sub(r"won't", "will not", phrase)
    phrase = re.sub(r"can\'t", "can not", phrase)
    phrase = re.sub(r"isn\'t", "is not", phrase)

    # general
    phrase = re.sub(r"n\'t", " not", phrase)
    phrase = re.sub(r"\'re", " are", phrase)
    phrase = re.sub(r"\'s", " is", phrase)
    phrase = re.sub(r"\'d", " would", phrase)
    phrase = re.sub(r"\'ll", " will", phrase)
    phrase = re.sub(r"\'t", " not", phrase)
    phrase = re.sub(r"\'ve", " have", phrase)
    phrase = re.sub(r"\'m", " am", phrase)
    return phrase

# we will also remove any punctuations and digits from our data
translation_table = str.maketrans('', '', string.punctuation + string.digits)


# + id="O1STbZVcQ78D"
def preprocess(items):
    results = []
    wnl = WordNetLemmatizer()
    nltk.download('wordnet')

    for item in items:
        # a. replace hyphens with space
        item = item.lower().replace('-', ' ') 

        # b. remove newlines, tabs
        item = item.replace('\n', '').replace('\t', '') 

        # c. remove punctuations and digits
        item = item.translate(translation_table) 

        # d. remove html and url tags from text
        item = re.sub(r"http\S+", "", item)
        item = BeautifulSoup(item, 'lxml').get_text()

        # e. decontract words
        item = decontracted(item) 

        # f. remove stop words
        item = remove_stopwords(item) # genism defined
        for stop_word in to_remove: # custom stop words
            item = item.replace(stop_word, '')

        # g. remove words with length lesser than three since most often they're stop words
        # h. and lemmatize
        item = [wnl.lemmatize(word) for word in item.split() if len(word) > 2]
        
        results.append(item)
    
    return results


# + colab={"base_uri": "https://localhost:8080/"} id="assD4dNoQ78E" outputId="7629d29f-1aa6-4271-d506-487ea860a919"
instructions = recipes.instructions.values.tolist()
ingredients = [', '.join(ingredient) for ingredient in recipes.ingredients.values]

train_data = instructions + ingredients
train_data[:5]

# + colab={"base_uri": "https://localhost:8080/"} id="dlQqAdpcQ78F" outputId="cd75bf6c-e5ad-4e9d-ce2f-4d605805c78c"
train_data_preprocessed = preprocess(train_data)
train_data_preprocessed[:2]

# + id="q9ev3OeaQ78G"
# save preprocessed train data
with open('data/train_data.pkl', 'wb') as f:
    pickle.dump(train_data_preprocessed, f)

# save the raw data too
with open('data/train_data_raw.pkl', 'wb') as f:
    pickle.dump(train_data, f)

recipes.to_csv('recipes.csv', index=False)

# + colab={"base_uri": "https://localhost:8080/"} id="1tzl3Od1sTg-" outputId="7597d0f6-aca9-4244-c3b2-c19d7b803116"
from google.colab import drive
drive.mount('/content/gdrive')

# + id="hNrYcT7JsiAA"
# !cp -r data/train_data_raw.pkl /content/gdrive/MyDrive/colab/xlabs/
# !cp -r data/train_data.pkl /content/gdrive/MyDrive/colab/xlabs/

StarcoderdataJupyterScriptsDedupFiltered