<jupyter_start><jupyter_text>1: [Diversity](diversity) 2: [Fairness](fairness) 3: [Consistency](consistency)<jupyter_code>import pandas as pd
import numpy as np
import locale
locale.setlocale(locale.LC_ALL, 'en_US')
from collections import Counter
import matplotlib.pyplot as plt
import string
import random
df = pd.read_csv('Data.csv')

#Get total views
videoKeys = list(df.columns)
videoKeys.remove("Unnamed: 0")
df['sumVideoViews'] = df[videoKeys].sum(axis=1)
#Extract Country,Age and Gender
df['country'] = df['Unnamed: 0'].str.split("_",expand=True)[0]
df['age'] = df['Unnamed: 0'].str.split("_",expand=True)[1]
df['gender'] = df['Unnamed: 0'].str.split("_",expand=True)[2]

df = df.drop('Unnamed: 0',axis=1)
print("Total Views",df[videoKeys].sum(axis=0).sum())
df.loc[df.age=="65-",'age'] = "65+" #Minor Fix
df.groupby("age")["sumVideoViews"].sum().sort_values(ascending=False)
#Top 10 countries by view
df.groupby("country")["sumVideoViews"].sum().sort_values(ascending=False)[:10]
dralgos = ["Clustering","PCA","NMF","LDA","SE","UMAP"]
def fixAge(temp): #Change Age Format
    for i in range(temp.iloc[1,1:].shape[0]):
        if int(temp.iloc[1,1+i]) <= 17:
            temp.iloc[1,1+i] = "13-17"
        elif int(temp.iloc[1,1+i]) > 17 and int(temp.iloc[1,1+i]) <= 24:
            temp.iloc[1,1+i] = "18-24"
        elif int(temp.iloc[1,1+i]) > 24 and int(temp.iloc[1,1+i]) <= 34:
            temp.iloc[1,1+i] = "25-34"
        elif int(temp.iloc[1,1+i]) > 34 and int(temp.iloc[1,1+i]) <= 44:
            temp.iloc[1,1+i] = "35-44"
        elif int(temp.iloc[1,1+i]) > 44 and int(temp.iloc[1,1+i]) <= 54:
            temp.iloc[1,1+i] = "45-54"
        elif int(temp.iloc[1,1+i]) > 54 and int(temp.iloc[1,1+i]) <= 64:
            temp.iloc[1,1+i] = "55-64"
        elif int(temp.iloc[1,1+i]) > 64:
            temp.iloc[1,1+i] = "65+"

    return temp
df["country"].nunique()
ccdf = pd.read_csv("CountryCode.csv") #Read country code DF
for i in range(df.shape[0]): #Change Country codes to full country names
    if df["country"].iloc[i] == "NA": #Pandas converted "NA" to "null", when we read the above csv file. So just fixing that.
        df["country"].iloc[i] = "Namibia"
    else:
        df["country"].iloc[i] = ccdf.loc[ccdf["Code"]==df["country"].iloc[i],'Name'].iloc[0]
df["country"].nunique()
totalViews = df["sumVideoViews"].sum()


#Append all personas in a list
dfs = []
for dralgo in dralgos:
    for i in range(3):
        temp = pd.read_excel("personas/"+dralgo+".xlsx",sheet_name=2-i)
        temp = fixAge(temp)
        dfs.append([temp,temp.shape[1]-1,dralgo])

#Create a CSV file to store the results
outputDF = pd.DataFrame(columns=["Dimensionality Reduction Algorithm","Number of Personas","GroupName","Subject","Percentage in Original Data","Unique Values in Original Data","Count in Persona Set","Percentage in Persona set","Total Unique Values in Persona Set","Statistical Parity"])<jupyter_output><empty_output><jupyter_text>Diversity<jupyter_code>#Create a CSV file to store the results
tempdf = pd.DataFrame(columns=["Dimensionality Reduction Algorithm","Number of Personas","GroupName","Unique Values in Original Data","Total Unique Values in Persona Set"])
for cdf in dfs:
    tempdf = tempdf.append(pd.DataFrame([[cdf[2],cdf[1],"Gender",df["gender"].nunique(),cdf[0].iloc[2,1:].nunique()]], columns=tempdf.columns))

for cdf in dfs:
    tempdf = tempdf.append(pd.DataFrame([[cdf[2],cdf[1],"Age",df["age"].nunique(),cdf[0].iloc[1,1:].nunique()]], columns=tempdf.columns))

for cdf in dfs:
    tempdf = tempdf.append(pd.DataFrame([[cdf[2],cdf[1],"Country",df["country"].nunique(),cdf[0].iloc[3,1:].nunique()]], columns=tempdf.columns))
diversitydf = tempdf.groupby(["Dimensionality Reduction Algorithm","Number of Personas","GroupName"], as_index=False)["Total Unique Values in Persona Set"].first()
diversitydf["persona_group"] = "coverage"+"_"+diversitydf["GroupName"] + "_" + diversitydf["Number of Personas"].astype(str)
diversitydf = diversitydf.drop(["Number of Personas","GroupName"],axis=1)
diversitydf = diversitydf.pivot(index='persona_group', columns='Dimensionality Reduction Algorithm', values='Total Unique Values in Persona Set')
diversitydf = diversitydf.reset_index().rename_axis(None, axis=1)
diversitydf
uniqueValues = tempdf.groupby(["GroupName"])["Unique Values in Original Data"].first()
for group in ["Age","Country","Gender"]:
    for numPersonas in [5,10,15]:
        persona_group = "coverage_"+group+"_"+str(numPersonas)
        diversitydf = diversitydf.append(pd.DataFrame([[persona_group+"_percentage","","","","","",""]], columns=diversitydf.columns))
        for algorithm in diversitydf.columns[1:]:
            uniqueNumInPersona = int(diversitydf.loc[diversitydf["persona_group"]==persona_group,algorithm].iloc[0])
            uniqueNumInData = uniqueValues[group]
            diversitydf[algorithm].iloc[diversitydf.shape[0]-1] = uniqueNumInPersona/uniqueNumInData*100
TotalUniqueValues = df["country"].nunique()+df["age"].nunique()+df["gender"].nunique()
TotalUniqueValues
for numPersona in [5,10,15]:
    total = np.zeros((len(dralgos)))
    for group in ["Country","Age","Gender"]:
        personaGroup = "coverage_"+group+"_"+str(numPersona)
        total = diversitydf.loc[diversitydf["persona_group"]==personaGroup].iloc[0,1:].values + total
    diversitydf = diversitydf.append(pd.DataFrame([["coverage_All_"+str(numPersona)]+total.tolist() ], columns=diversitydf.columns))
    
    total_perc = total/TotalUniqueValues
    diversitydf = diversitydf.append(pd.DataFrame([["coverage_All_"+str(numPersona)+"__percentage"]+total_perc.tolist() ], columns=diversitydf.columns))
diversitydf
diversitydf.to_csv("diversitydf.csv",index=False)<jupyter_output><empty_output><jupyter_text>Fairness Calculating Fairness for age, gender and country.<jupyter_code>#Create a CSV file to store the results
outputDF = pd.DataFrame(columns=["Dimensionality Reduction Algorithm","Number of Personas","GroupName","Subject","Percentage in Original Data","Unique Values in Original Data","Count in Persona Set","Percentage in Persona set","Total Unique Values in Persona Set","Statistical Parity"])
#Calculate fairness for gender
totalMale = df.loc[df["gender"]=="male"].groupby("gender")["sumVideoViews"].sum().values[0]
for cdf in dfs:
    cTotal = (cdf[0].iloc[2,1:].values=="male").sum()
    outputDF = outputDF.append(pd.DataFrame([[cdf[2],cdf[1],"Gender","Male",np.round(totalMale/totalViews*100,3),df["gender"].nunique(),cTotal,np.round(cTotal/cdf[1]*100,3),cdf[0].iloc[2,1:].nunique(),np.round(cTotal/cdf[1]-totalMale/totalViews,3)]], columns=outputDF.columns))

#Calculate fairness for Age
    
for age in ["25-34","35-44","18-24","45-54","55-64","65+","13-17"]:
    totalAge = df.loc[df["age"]==age].groupby("age")["sumVideoViews"].sum().values[0]
    for cdf in dfs:
        cTotal = (cdf[0].iloc[1,1:].values==age).sum()
        outputDF = outputDF.append(pd.DataFrame([[cdf[2],cdf[1],"Age",age,np.round(totalAge/totalViews*100,3),df["age"].nunique(),cTotal,np.round(cTotal/cdf[1]*100,3),cdf[0].iloc[1,1:].nunique(),np.round(cTotal/cdf[1]-totalAge/totalViews,3)]], columns=outputDF.columns))

#Calculate fairness for Country

df = df.loc[~df["country"].isnull()]

for country in df.country.unique().tolist():

    totalCountry = df.loc[df["country"]==country].groupby("country")["sumVideoViews"].sum().values[0]
    for cdf in dfs:
        cTotal = (cdf[0].iloc[3,1:].values==country).sum()
        outputDF = outputDF.append(pd.DataFrame([[cdf[2],cdf[1],"Country",country,np.round(totalCountry/totalViews*100,3),df["country"].nunique(),cTotal,np.round(cTotal/cdf[1]*100,3),cdf[0].iloc[3,1:].nunique(),np.round(cTotal/cdf[1]-totalCountry/totalViews,3)]], columns=outputDF.columns))

outputDF['Fairness'] = np.abs(outputDF['Statistical Parity'])
outputDF.to_csv("stat_parity.csv",index=False)
#Save country fairness data
countryfairness = outputDF.loc[outputDF["GroupName"]=="Country"].groupby(['Dimensionality Reduction Algorithm','GroupName','Number of Personas'],as_index=False)['Fairness'].mean()

countryfairness.to_csv('Country_Stat_Parity_Mean.csv',header=True)

countryfairness
#Save Age fairness data
agefairness = outputDF.loc[outputDF["GroupName"]=="Age"].groupby(['Dimensionality Reduction Algorithm','GroupName','Number of Personas'],as_index=False)['Fairness'].mean()

agefairness.to_csv('Age_Stat_Parity_Mean.csv',header=True)

agefairness
#Save Gender fairness data
genderfairness = outputDF.loc[outputDF["GroupName"]=="Gender"].groupby(['Dimensionality Reduction Algorithm','GroupName','Number of Personas'],as_index=False)['Fairness'].mean()

genderfairness.to_csv('Gender_Stat_Parity_Mean.csv',header=True)

genderfairness<jupyter_output><empty_output><jupyter_text>Consistency<jupyter_code>consistencyDF = pd.DataFrame(columns=["Algorithm","Consistency Score"])
def randomString(stringLength): #Generate a random string
    letters = string.ascii_lowercase
    return ''.join(random.choice(letters) for i in range(stringLength))

def common(lst1, lst2): 
    #This function finds the count of common values between two lists.
    #But we also do not want to use a single value from first list to match two values in the second list.
    #Example:
    #List1 = [1,1,2,3,4]
    #List2 = [1,1,1,2,2,3,4]
    #Score = 5. Because Two ones in list1 match two times. One two in list1 match one time. 3 and 4 match 1 time. So total 5 matches
    counter = 0
    lst11,lst22= lst1.copy(),lst2.copy() #Copy them, so we don't overwrite the original 
    for i in range(len(lst11)):
        for j in range(len(lst22)):
            if lst11[i]==lst22[j]:
                counter +=1
                lst11[i] = randomString(10000) #We don't want to count it again. So we just assign a random value to it.
                lst22[j] = randomString(10000)
                break
    return counter
for dralgo in dralgos: #Need to process files of all algorithms
    uniqueValues = []
    for i in range(3): #Need to read the file 3 times with all 3 sheets. (5,10,15)
        temp = pd.read_excel("personas/"+dralgo+".xlsx",sheet_name=2-i)
        uniqueValues_tmp = []
        for i in range(temp.shape[1]-1):
            #Join Age, Gender and country. 
            value = str(temp.iloc[1,1+i:2+i].values[0])+temp.iloc[2,1+i:2+i].values[0]+temp.iloc[3,1+i:2+i].values[0]
            uniqueValues_tmp.append(value)
        uniqueValues.append(uniqueValues_tmp)
      

    # Find common between 5 persona set and 10
    score = common(uniqueValues[0], uniqueValues[1])/5
    # Find common between 5 persona set and 15
    score += common(uniqueValues[0], uniqueValues[2])/5
    # Find common between 10 persona set and 15
    score += common(uniqueValues[1], uniqueValues[2])/10
    score = score/3
    
    consistencyDF = consistencyDF.append(pd.DataFrame([[dralgo,score]], columns=consistencyDF.columns))
consistencyDF
consistencyDF.to_csv("consistencyDF.csv",index=False)<jupyter_output><empty_output><jupyter_text>DFC in one table<jupyter_code>totalFairness = pd.concat([agefairness,countryfairness,genderfairness],axis=0)
totalFairness = totalFairness.groupby(["Dimensionality Reduction Algorithm","Number of Personas"],as_index=False)["Fairness"].mean()
totalFairness["Consistency"] = 0
for i in range(totalFairness.shape[0]):
    totalFairness["Consistency"].iloc[i] = consistencyDF.loc[consistencyDF["Algorithm"]==totalFairness["Dimensionality Reduction Algorithm"].iloc[i]]["Consistency Score"].iloc[0]
totalFairness["Diversity"] = 0
totalFairness
for algo in dralgos:
    totalFairness.loc[(totalFairness["Dimensionality Reduction Algorithm"]==algo) & (totalFairness["Number of Personas"]==5),'Diversity'] = diversitydf.loc[diversitydf["persona_group"]=="coverage_All_5__percentage"][algo].iloc[0]
    totalFairness.loc[(totalFairness["Dimensionality Reduction Algorithm"]==algo) & (totalFairness["Number of Personas"]==10),'Diversity'] = diversitydf.loc[diversitydf["persona_group"]=="coverage_All_10__percentage"][algo].iloc[0]
    totalFairness.loc[(totalFairness["Dimensionality Reduction Algorithm"]==algo) & (totalFairness["Number of Personas"]==15),'Diversity'] = diversitydf.loc[diversitydf["persona_group"]=="coverage_All_15__percentage"][algo].iloc[0]
totalFairness.to_csv("DFC.csv",index=False)
totalFairness<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup