# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
import numpy as np
import pyspark
import os

from pyspark import SparkConf , SparkContext
from pyspark.sql import SparkSession, SQLContext

from pyspark.sql.types import *
import pyspark.sql.functions as F
from pyspark.sql.functions import udf, col

from pyspark.ml.regression import LinearRegression
from pyspark.mllib.evaluation import RegressionMetrics

from pyspark.ml.tuning import ParamGridBuilder, CrossValidator, CrossValidatorModel
from pyspark.ml.feature import VectorAssembler, StandardScaler
from pyspark.ml.evaluation import RegressionEvaluator
# -

import seaborn as sns
import matplotlib.pyplot as plt

from pyspark import SparkContext
sc =  SparkContext()

from pyspark import SparkConf
from pyspark import SparkContext

conf = SparkConf().setAppName("SparkRDD").setMaster("local")
sc1 = SparkContext(conf=conf)

sc1

values = [1,2,3,4,5]
rdd = sc1.parallelize(values)

rdd.take(2)

rdd= sc1.textFile("../trgc1000.csv")

rdd.take(1)



aba = sc1.parallelize(range(1,10000,2))
aba.persist()

text_file = sc1.textFile("../trgc1000.csv")
text_file.cache()

text_file

# +
#Visualization
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

pd.set_option("display.max_columns",200)
pd.set_option("display.max_colwidth",400)

from matplotlib import rcParams
sns.set(context = "notebook",style= "whitegrid",rc = {"figure.figsize":(18,4)})
rcParams["figure.figsize"]= 18,4

# %matplotlib inline
# %config InlineBackend.figure_format = "retina"
# -

rnd_seed = 23
np.random.seed = rnd_seed
np.random.set_state = rnd_seed

# ## 2. Creating the Spark Session

spark = SparkSession.builder.master("local[2]").\
appName("Linear-Regression-California-Housing").getOrCreate()

spark

sc= spark.sparkContext
sc

sqlContext = SQLContext(spark.sparkContext)
sqlContext

HOUSING_DATA = "cal_housing.data"

# +
#define the schema, corresponding to a line in the csv data
schema = StructType([
    StructField("long", FloatType(),nullable = True),
    StructField("lat",FloatType(),nullable=True),
    StructField("medage",FloatType(),nullable=True),
    
])
# -





StarcoderdataJupyterScriptsDedupFiltered