# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# + id="T4uqqLQm4gZM"
import pandas as pd
import numpy as np

# + id="dbuXOrGw41jf"
cluster = 4

# + id="T0FbuH-n4gZO"
country_data = pd.read_csv("synthesized_data.csv")

# + colab={"base_uri": "https://localhost:8080/"} id="ROU5nRbx4gZO" outputId="aedb0fb3-6609-4eb4-a7a2-fd22e943598c"
country_data.shape

# + colab={"base_uri": "https://localhost:8080/"} id="K7oA-Lct4gZP" outputId="80139f3c-942b-4a1d-ad81-2f21c516a2a8"
country_data.isnull().sum()

# + id="4Cg2HH-54gZP"
country_data['Primary enrollment rate'] = country_data['Primary enrollment'] / country_data['Population (thousands)']
country_data['Secondary enrollment rate'] = country_data['Secondary enrollment'] / country_data['Population (thousands)']
country_data['Tertiary enrollment rate'] = country_data['Tertiary enrollment'] / country_data['Population (thousands)']
country_data['Lower secondary repeaters rate'] = country_data['Lower secondary repeaters count'] / country_data['Population (thousands)']
useful_data = country_data.loc[:, ~country_data.columns.isin(['Country', 'Primary enrollment', 'Secondary enrollment', 'Tertiary enrollment', 'Lower secondary repeaters count', 'Primary repeaters count', 'Average income', 'Average income currency'])]

# + colab={"base_uri": "https://localhost:8080/"} id="aB1gheN-4gZQ" outputId="1472020b-9c8c-4a7a-f1bb-44589ff97570"
useful_data.isnull().sum()

# + id="Dnh3HeCa4gZQ"
from sklearn.impute import SimpleImputer

imputer = SimpleImputer(missing_values=np.nan, strategy='median')
imputed_data = pd.DataFrame(imputer.fit_transform(useful_data))
standardized_data = (imputed_data-imputed_data.mean())/imputed_data.std()

# + id="jzRFoVS94gZQ"
from sklearn.decomposition import PCA
pca_countries = PCA(n_components=2)
principalComponents_countries = pca_countries.fit_transform(standardized_data)

# + colab={"base_uri": "https://localhost:8080/", "height": 282} id="aI6cXxXT4gZQ" outputId="ea4e03cf-841d-4e50-b0db-bf0697e52b16"
import matplotlib.pyplot as plt
# %matplotlib inline

plt.scatter(principalComponents_countries[:, 0], principalComponents_countries[:, 1])

# + id="LWskfzdu4gZR"
from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=cluster)
cluster_vals = kmeans.fit_predict(standardized_data)

# + colab={"base_uri": "https://localhost:8080/", "height": 282} id="LlOYWggt4gZR" outputId="19b2f055-c900-4d0a-b235-7c7213a9cd66"
plt.scatter(principalComponents_countries[cluster_vals ==0,0], principalComponents_countries[cluster_vals == 0,1], s=100, c='red')
plt.scatter(principalComponents_countries[cluster_vals ==1,0], principalComponents_countries[cluster_vals == 1,1], s=100, c='black')
plt.scatter(principalComponents_countries[cluster_vals ==2,0], principalComponents_countries[cluster_vals == 2,1], s=100, c='blue')
plt.scatter(principalComponents_countries[cluster_vals ==3,0], principalComponents_countries[cluster_vals == 3,1], s=100, c='green')
plt.scatter(principalComponents_countries[cluster_vals ==4,0], principalComponents_countries[cluster_vals == 4,1], s=100, c='cyan')
plt.scatter(principalComponents_countries[cluster_vals ==5,0], principalComponents_countries[cluster_vals == 5,1], s=100, c='purple')
plt.scatter(principalComponents_countries[cluster_vals ==6,0], principalComponents_countries[cluster_vals == 6,1], s=100, c='orange')
plt.scatter(principalComponents_countries[cluster_vals ==7,0], principalComponents_countries[cluster_vals == 7,1], s=100, c='yellow')
plt.scatter(principalComponents_countries[cluster_vals ==8,0], principalComponents_countries[cluster_vals == 8,1], s=100, c='grey')
plt.scatter(principalComponents_countries[cluster_vals ==9,0], principalComponents_countries[cluster_vals == 9,1], s=100, c='brown')
plt.scatter(principalComponents_countries[cluster_vals ==10,0], principalComponents_countries[cluster_vals == 10,1], s=100, c='magenta')
plt.scatter(principalComponents_countries[cluster_vals ==11,0], principalComponents_countries[cluster_vals == 11,1], s=100, c='lime')
plt.scatter(principalComponents_countries[cluster_vals ==12,0], principalComponents_countries[cluster_vals == 12,1], s=100, c='pink')
plt.scatter(principalComponents_countries[cluster_vals ==13,0], principalComponents_countries[cluster_vals == 13,1], s=100, c='maroon')
plt.scatter(principalComponents_countries[cluster_vals ==14,0], principalComponents_countries[cluster_vals == 14,1], s=100, c='azure')

# + id="Z0N0gvRw4gZS"
cv_sr = pd.Series(cluster_vals)
clusters = []
for i in range(cluster):
    clusters.append(country_data['Country'][cv_sr == i].to_list())

# + colab={"base_uri": "https://localhost:8080/"} id="wvWor1974gZS" outputId="932b4784-4e81-46e0-b494-4ceb88e0b0a0"
clusters

# + colab={"base_uri": "https://localhost:8080/"} id="Dv3hrrf24gZS" outputId="daba4935-4820-4061-9fc9-bb31e1160c61"
education_data = standardized_data[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 20, 21, 22, 23]]
cluster_vals_ed = kmeans.fit_predict(education_data)
cv_sr_ed = pd.Series(cluster_vals_ed)
clusters_ed = []
for i in range(cluster):
    clusters_ed.append(country_data['Country'][cv_sr_ed == i].to_list())
clusters_ed

# + colab={"base_uri": "https://localhost:8080/"} id="5Pi9CnzM4gZS" outputId="064a3762-0b27-4dc7-f156-7dcec35fdc5a"
economic_data = standardized_data[[0, 14, 15, 16, 17, 18, 19]]
cluster_vals_ec = kmeans.fit_predict(economic_data)
cv_sr_ec = pd.Series(cluster_vals_ec)
clusters_ec = []
for i in range(cluster):
    clusters_ec.append(country_data['Country'][cv_sr_ec == i].to_list())
clusters_ec

# + id="kYHPl9EF4gZT"
import collections

ed_ec_combined = collections.defaultdict(list)

for i in range(cluster):
  ed_ec_combined[i] = collections.defaultdict(list)

for i in range(178):
  country = country_data['Country'][i]
  ed_ec_combined[cv_sr_ed[i]][cv_sr_ec[i]].append(country)

# + colab={"base_uri": "https://localhost:8080/"} id="P_999SEv4dhm" outputId="c20af508-7e81-49d2-fa7a-20e641c1b212"
ed_ec_combined

# + colab={"base_uri": "https://localhost:8080/", "height": 496} id="WN8PHvRf-0FA" outputId="9ae2c019-e29b-45d0-fae7-9c1913244074"
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(cv_sr_ec, cv_sr_ed)
print(cm)

from mlxtend.plotting import plot_confusion_matrix
 
fig, ax = plot_confusion_matrix(conf_mat=cm, figsize=(6, 6), cmap=plt.cm.Blues)
plt.xlabel('Economic Clusters', fontsize=16)
plt.ylabel('Education Clusters', fontsize=16)
plt.title('Confusion Matrix', fontsize=16)
plt.show()

# + colab={"base_uri": "https://localhost:8080/"} id="k70MHlxRH76q" outputId="a68fbdf9-3e55-4a79-ebc6-ac5effd73daa"
df = country_data[country_data['Country'] == ed_ec_combined[2][2][0]]

for entry in ed_ec_combined[2][2]:
  print(entry)
  df = df.append(country_data[country_data['Country'] == entry])

# + id="tFodBr_-IoQI"
index = [8, 15, 18, 30, 44, 45, 57, 58, 62, 70, 72, 79, 81, 94, 97, 117, 121, 130, 132, 134, 144, 154, 155, 159, 168, 169]



# + id="USF7KBXTNshH"
df = pd.DataFrame(columns=imputed_data.columns)

for i in index:
  df.loc[len(df.index)] = imputed_data.loc[i]

# + id="QBaWJsQFectL"
import csv

table = []
sig_table = []

with open("all_feature_comparison.csv", "w", newline='') as f:
  with open("best_feature_comparison.csv", "w", newline='') as g:
    w1 = csv.writer(f)
    w2 = csv.writer(g)
    table.append(['', 'Mean of Best Cluster', 'Mean of All Countries', 'Difference (%)'])
    sig_table.append(['', 'Mean of Best Cluster', 'Mean of All Countries', 'Difference (%)'])
    for i in range(24):
      diff = abs((imputed_data[i].mean() - df[i].mean()) / df[i].mean()) * 100
      row = [useful_data.columns[i], df[i].mean(), imputed_data[i].mean(), diff]
      table.append(row)
      if diff > 45:
        sig_table.append(row)
    w2.writerows(sig_table)

  w1.writerows(table)

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="ANqFABrcyiuV" outputId="f9837f03-2475-43f8-c5b2-6e6d7c6fbf7f"
from tabulate import tabulate

print(tabulate(table, headers='firstrow', tablefmt='fancy_grid', numalign="right"))
table.to_csv('all_feature_comparison.csv')

# + colab={"base_uri": "https://localhost:8080/"} id="GVOLXFwv_aND" outputId="fd7b7205-5c9e-4411-a12d-8fab6e4729c7"
print(tabulate(sig_table, headers='firstrow', tablefmt='fancy_grid', numalign="right"))

StarcoderdataJupyterScriptsDedupFiltered