# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### PySpark Introduction

# *If you receive a "name 'sc' is not defined" error message, stop pyspark and delete the metastore_db directory in the same directory as this Jupyter Notebook*

# **Python Version**

import sys
print(sys.version)

# **Spark Context**

print(sc)

# **Spark Version**

print(sc.version)

# **Creating a RDD**

testList = [25, 90, 81, 37, 776, 3320]
testData = sc.parallelize(testList)

type(testData)

testData.count()

testData.collect()

# ### Running a PySpark Application

# RDDs are distributed collections of items. They can be created from Hadoop (files in HDFS), from others RDDs transformation, from databases (relational and non-relational) or from local files.

rddFeelings = sc.textFile('aux/datasets/feelings.csv')

type(rddFeelings)

rddFeelings.count()

rddFeelings.take(5)

rddFeelingsUpperCase = rddFeelings.map(lambda letter: letter.upper())
rddFeelingsUpperCase.take(5)

rddFeelings.take(5)

rddFeelings.first()

linesWithSol = rddFeelings.filter(lambda line: 'Sol' in line)

type(linesWithSol)

linesWithSol.count()

# In the example below, the map() function determines the length of each line in the file, creating a RDD. Then, the reduce() function is called to find the line with the largest number of characters.<br/>
# The arguments for the map() and reduce() functions are anonymous functions created with lambda (from Python).

rddFeelings.map(lambda line: len(line.split())).reduce(lambda x, y: x if (x > y) else y)

# ### MapReduce Operations

countWords = rddFeelings.flatMap(lambda line: line.split()).map(lambda word: (word, 1)).reduceByKey(lambda x, y: x + y)

countWords.collect()

# Access Spark Monitor at: http://localhost:4040 /

StarcoderdataJupyterScriptsDedupFiltered