# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import joblib
import seaborn as sns
sns.set_palette(sns.color_palette("hls", 11))

# %matplotlib inline
# -

data = joblib.load('./representations.joblib')

data.keys()

# +
flatten = lambda l: [item for sublist in l for item in sublist]

X = []
labels = []
for label, key in enumerate(data):
    for _ in data[key][0]:
        labels.append(label)
    for x in data[key]:
        X.extend(x)
# -

len(X)

X[0].shape

tsne = TSNE(n_components=2)
X_tsne = tsne.fit_transform(X)

x, y = zip(*X_tsne)

cm = []
colors = sns.color_palette()
for label in labels:
    cm.append(colors[label])

plt.scatter(x,y,color=cm, marker='.')



StarcoderdataJupyterScriptsDedupFiltered