# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Barneybean/Text_Indexing/blob/master/MapReduce.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="Ryy1SVri3PA_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 224} outputId="e643d009-f659-4c96-af8d-7afed6103aeb"
#Method - April, modified - William
from functools import reduce
import numpy as np

import operator
import pandas as pd

book_by_page = [["Mary", "Lamb", "Had", "A", "A"],["A","Little", "Lamb"],["Joseph","Had","A","Baby"]]

#remove duplicate in a page
book_by_page = list(map(lambda x: set(x), book_by_page))

#see tuples in testing eg:[(0, {'A', 'Had', 'Mary', 'Lamb'})], this ste is to assign page number to each word on each page {i:[(x[0]+1)]} => {'Had': [1]}
map_list = list(map(lambda x: list({i:[(x[0]+1)]} for i in x[1]), enumerate(book_by_page)))
print("map_list", map_list)
print("--------")

#map_list = list(map(lambda x: {i:[(x[0]+1)] for i in x[1]} , enumerate(book_by_page)))

flat_map_list = reduce(operator.concat, map_list)
print("flat_map_list", flat_map_list)
print("--------")

flat_map_list

def merge_dict(x,y):
    if list(y.keys())[0] in list(x.keys()):
        x[list(y.keys())[0]].append(list(y.values())[0][0])
    else:
        x.update(y)
    return x

reduce(merge_dict,flat_map_list)

# + id="36vmfES-qgR5" colab_type="code" outputId="fc9d2b84-a4b7-4e6e-ceaf-a9eea8fa2ab0" colab={"base_uri": "https://localhost:8080/", "height": 102}
# Method1 - William
from functools import reduce 
book_by_page = [['Mary', 'Lamb', 'Had', 'A', "A"],['Little', 'Lamb'],["Joseph","Had","A","Baby"]]

#get unique words 
words = reduce(lambda x, y: x+y, book_by_page)
unique_words = list(set(words))
print(f'unique words in the book are: \n{unique_words}')

def map_fn(data):
    idx, el = data
    dict = {}
    dict[idx+1]=el
    return dict
  
book = list(map(map_fn, enumerate(book_by_page)))
# [{1: ['A', 'Had', 'Mary', 'Lamb']}, {2: ['Lamb', 'Little']}]
print(book)

# create result dictionary
result = {}
for x in unique_words:
  result[x] = []
# print(result)

def search_words(word):
  # count is used as key of each item in the book list
  page_num = 1
  for item in book: 
    if word in item[page_num]:
      #if word occurs in the page then append page number to result list of the word
      result[word].append(page_num)
      page_num += 1
    else:
      page_num += 1

     
list(map(search_words, unique_words))

print(f"index of each word is: \n{result}")

# + id="OWuIbQncqUpI" colab_type="code" outputId="c3e272e9-012b-4c74-f998-b233e667da5d" colab={"base_uri": "https://localhost:8080/", "height": 119}
#testing
a = {"mary":[1]}
b=[1,2,2,3]
a["bob"]=b
print(a)

print(list(a.keys())[0])

#testing
b.index(2)
#get unique values in a list
set(b)

def check_keys(dict): 
    if "mary" not in dict.keys():
        print(1)
    else:
        print(2)
    
check_keys(a)  

#testing - get index in map using emulate
my_list = [2,3,5,7]

def map_fn(data):
    idx, el = data
    dict = {}
    dict[el]=[idx+1]
    return dict

list(map(map_fn, enumerate(my_list)))

# tuples
tuples = list(enumerate(book_by_page))
print("tuples", tuples)

print("-----")

print(tuples[0][0], "and", tuples[0][1])
  

# + id="Wq_MM2udqn61" colab_type="code" outputId="95212bba-ffd1-43f6-d286-5d577ad0ff9e" colab={"base_uri": "https://localhost:8080/", "height": 224}


# + id="GdQMBqNi1Sd0" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered