# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# !pip install pyspark


from pyspark.sql import SparkSession
spark=SparkSession.builder.appName('App3').getOrCreate()

spark

import pandas as pd
pd.read_csv('titanic1.csv')

from pyspark.sql import SparkSession

spark=SparkSession.builder.appName('App3').getOrCreate()

spark.read.option('header','true').csv('titanic1.csv').show()

df_pyspark=spark.read.option('header','true').csv('titanic1.csv',inferSchema=True)
df_pyspark.show()

df_pyspark.printSchema()

# +
from pyspark.sql.types import IntegerType

df_pyspark = df_pyspark.withColumn("Age", df_pyspark["Age"].cast(IntegerType()))
df_pyspark = df_pyspark.withColumn("Fare", df_pyspark["Fare"].cast(IntegerType()))
# -

df_pyspark.printSchema()

type(df_pyspark)

# +
# drop COLUMNS
# -

df_pyspark.drop('Age').show()

# +
# drop NULL (na) values
# -

df_pyspark.select().count()

df_pyspark.na.drop().select().count()

df_pyspark.na.drop().select().count() - df_pyspark.select().count()

df_pyspark.na.drop().show()

# +
# any & how, drop NULL (na) values
# -

df_pyspark.na.drop(how='all').select().count() - df_pyspark.select().count()

df_pyspark.na.drop(how='all').show()

df_pyspark.na.drop(how='any').select().count() - df_pyspark.select().count()

df_pyspark.na.drop(how='any').show()

# +
# threshold, drop NULL (na) values
# -

df_pyspark.na.drop(how='any', thresh=2).select().count() - df_pyspark.select().count()

df_pyspark.na.drop(how='any', thresh=2).show()

df_pyspark.na.drop(how='any', thresh=1).select().count() - df_pyspark.select().count()

df_pyspark.na.drop(how='any', thresh=3).select().count() - df_pyspark.select().count()

# +
# subset, drop NULL (na) values
# -

df_pyspark.na.drop(how='any', subset=['Sex']).select().count() - df_pyspark.select().count()

df_pyspark.na.drop(how='any', subset=['Sex']).show()

# +
# filling missing values - fill all, fill columns,
# -

df_pyspark.na.fill('Missing Vlues').show()

df_pyspark.na.fill('Missing Vlues',['Sex']).show()

df_pyspark.na.fill('Missing Vlues',['Age','Fare']).show()

# +
# Imputer function
# -

df_pyspark.show()

from pyspark.ml.feature import Imputer

df_pyspark.printSchema()

# +
from pyspark.ml.feature import Imputer

df_pyspark = df_pyspark.withColumn("Age", df_pyspark['Age'].cast('double')).withColumn('Fare', df_pyspark['Fare'].cast('double'))

imputer = Imputer(
    inputCols=['Age', 'Fare'],
    outputCols=['Age', 'Fare'])

imputer.fit(df_pyspark).transform(df_pyspark)
# -

df_pyspark.printSchema()

df_pyspark.describe().show()

# +
# get the Mean of a column

df_pyspark.agg({'Fare': 'mean'}).show()
# -

df_pyspark.agg({'Age': 'avg'}).show()

# +
# get the Standard Deviation of a column

df_pyspark.agg({'Fare': 'stddev'}).show()
# -

df_pyspark.agg({'Fare': 'variance'}).show()

imputer = Imputer(
    inputCols=['Age', 'Fare'],
    outputCols=["{}_imputed".format(c) for c in ['Age', 'Fare'] ]
    ).setStrategy("mean")


# +
# add imputation cols to df
# -

imputer.fit(df_pyspark).transform(df_pyspark).show()

StarcoderdataJupyterScriptsDedupFiltered