<jupyter_start><jupyter_text>Configure parameters<jupyter_code># Path to dataset file
#data_path='/data/biodata/Iris/'
%store -r path


# Sample of train and test dataset
train_sample = 0.7
test_sample = 0.3
# Create Spark Session
spark = SparkSession.builder \
        .master("local[8]") \
        .appName("MachineLearningIris") \
        .getOrCreate()

# Enable Arrow-based columnar data transfers
#spark.conf.set("spark.sql.execution.arrow.enabled", "true")
# Load Iris CSV dataset to Spark Dataframe
orig_data = spark.read.format("csv").options(sep=',',header='true',inferschema='true').\
            load(path)

print("Original Dataframe read from CSV file")
#orig_data.dtypes
orig_data.show(5)
# ML libraries doesn't accept string column => everything should be numeric! 
# create a numeric column "label" based on string column "class" 

indexer = StringIndexer(inputCol="class", outputCol="label").fit(orig_data)
label_data = indexer.transform(orig_data)

# Save the inverse map from numeric "label" to string "class" to be used further in response
labelReverse = IndexToString().setInputCol("label")

# Show labeled dataframe with numeric lable
print("Dataframe with numeric lable")
label_data.show(5)
# Drop string column "class", no string column
label_data = label_data.drop("class")

# Most Machine Learning Lib inpute 2 columns: label (output) and feature (input)
# The label column is the result to train ML algorithm 
# The feature column should join all parameters as a Vector

# Set the column names that is not part of features list
ignore = ['label']
# list will be all columns parts of features
list = [x for x in label_data.columns if x not in ignore]

# VectorAssembler mount the vector of features
assembler = VectorAssembler(
            inputCols=list,
            outputCol='features')

# Create final dataframe composed by label and a column of features vector
data = (assembler.transform(label_data).select("label","features"))

print("Final Dataframe suitable to classifier input format")
#data.printSchema()
data.show(5)
# Split ramdomly the dataset into train and test group
# [0.7,0.3] => 70% for train and 30% for test
# [1.0,0.2] => 100% for train and 20% for test, not good, acuracy always 100%
# [0.1,0.02] => 10% for train and 2% for test, if big datasets
# 1234 is the random seed

(train, test) = data.randomSplit([train_sample, test_sample], 1234)
start_time_nb =  time.time()
# create the trainer and set its parameters
trainer = NaiveBayes(smoothing=1.0, modelType="multinomial")

#trainer = LogisticRegression(maxIter=10, tol=1E-6, fitIntercept=True)
#trainer = RandomForestClassifier(labelCol="indexedLabel", featuresCol="indexedFeatures", numTrees=10)

# train the model and get the result
model = trainer.fit(train)
result_nb = model.transform(test)
# compute accuracy on the test set against model
evaluator = MulticlassClassificationEvaluator(labelCol="label", predictionCol="prediction",\
            metricName="accuracy")

accuracy_nb = evaluator.evaluate(result_nb) * 100
time_nb = time.time() - start_time_nb

print("Naive Bayes: accuracy = %3.1f %%" % accuracy_nb)
print("Naive Bayes: time = %3.3f s" % time_nb)
print("Naive Bayes Final Result")
result_nb.show()
print("Naive Bayes final result with name of class")
labelReverse.transform(result_nb).show()<jupyter_output>Naive Bayes final result with name of class
+-----+-----------------+--------------------+--------------------+----------+----------------------------------+
|label|         features|       rawPrediction|         probability|prediction|IndexToString_982c59433006__output|
+-----+-----------------+--------------------+--------------------+----------+----------------------------------+
|  0.0|[4.3,3.0,1.1,0.1]|[-9.8947865959498...|[0.73694376226986...|       0.0|                       Iris-setosa|
|  0.0|[4.4,2.9,1.4,0.2]|[-10.798200610812...|[0.65275740620864...|       0.0|                       Iris-setosa|
|  0.0|[4.4,3.0,1.3,0.2]|[-10.714490841463...|[0.68253047838642...|       0.0|                       Iris-setosa|
|  0.0|[4.8,3.1,1.6,0.2]|[-11.685557246997...|[0.66369141823549...|       0.0|                       Iris-setosa|
|  0.0|[5.0,3.5,1.6,0.6]|[-13.716429687974...|[0.60272102805515...|       0.0|                       Iris-setosa|
|  0.0|[5.0,3.6,1.4,0.2]|[-11.987255175816..[...]
StarcoderdataJupyterStructuredCleanDedup