# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import findspark
findspark.init()
from pyspark.sql import SparkSession

spark = SparkSession.builder.appName('spoiled_food').master('local[4]').getOrCreate()

# ### Use RF to Determine Which Chemical Causes Early Spoiling

# +
# we probably don't need machine learning to solve this
# train test split probably doesn't matter much
# we are really interested in the entropy methods of decision trees
# -

data = spark.read.csv('../data/dog_food.csv', header=True, inferSchema=True)

data.printSchema()

data.show()

data.describe().show()

# check for missing data
data.na.drop().describe().show()

# ### Create Features

from pyspark.ml.feature import VectorAssembler

assembler = VectorAssembler(
    inputCols=['A', 'B', 'C', 'D'],
    outputCol='features')

with_features = assembler.transform(data).select('features', 'Spoiled')

with_features.printSchema()

with_features.head(5)

with_features.filter(with_features['Spoiled'] == 1).count()

with_features.filter(with_features['Spoiled'] == 0).count()

# +
# we see there is some imbalance, some foods are not spoiling
# -

# ### Train RF Classifier with All Data

# +
# we don't need train / test split, just trying to find important attributes
# -

from pyspark.ml.classification import RandomForestClassifier

rfc = RandomForestClassifier(numTrees=100, labelCol='Spoiled')

rfc_model = rfc.fit(with_features)

rfc_model.featureImportances

# +
# we see that the 3rd attribute (C) provides 93.44% of the Spoilage
# -

# ### Does Attribute C Indicare Spoilage?

spoiled_data = data.filter(data['Spoiled'] == 1).select('C', 'Spoiled')

non_spoiled_data = data.filter(data['Spoiled'] == 0).select('C', 'Spoiled')

spoiled_data.describe().show()

non_spoiled_data.describe().show()

# - spoiled data has chemical C with mean of 11.9
# - non-spoiled data has chemical C with mean of 8.0
# - looks like we need to reduce chemical C to <= 9!



StarcoderdataJupyterScriptsDedupFiltered