# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
from numpy import array
from math import sqrt

from pyspark.mllib.clustering import KMeans, KMeansModel
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
# -

data = sc.textFile("../data/mllib/kmeans_data.txt")
parsedData = data.map(lambda line: array([float(x) for x in line.split(' ')]))

print(parsedData.collect())

clusters = KMeans.train(parsedData, 2, maxIterations=10, initializationMode="random")

print(clusters.centers)


# +
def error(point):
    center = clusters.centers[clusters.predict(point)]
    return sqrt(sum([x**2 for x in (point - center)]))

WSSSE = parsedData.map(lambda point: error(point)).reduce(lambda x, y: x + y)
print("Within Set Sum of Squared Error = " + str(WSSSE))

StarcoderdataJupyterScriptsDedupFiltered