# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .r
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: R
#     name: ir
# ---

# + [markdown] colab_type="text" id="view-in-github"
# <a href="https://colab.research.google.com/github/ValentineKarimi/Carrefour-Kenya-Analysis-/blob/main/Feature_Selection.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="GU_C0ga6S3Jz"
# ##**Reading and checking the data**

# + colab={"base_uri": "https://localhost:8080/"} id="QSrV2Z0xR9Qw" outputId="994c7196-e217-444a-d099-06036a0dd97c"
# dowloading tidyverse packages and library 
install.packages('tidyverse')
library(tidyverse)

# Installing our ggbiplot visualisation package

library(devtools)
install_github("vqv/ggbiplot")


# Then Loading our ggbiplot library
#  
library(ggbiplot)

# + colab={"base_uri": "https://localhost:8080/"} id="78qhcb9RSB30" outputId="e7f52ad7-26e4-44eb-ccc7-d0291597fe3a"
#Read the dataset 
salesdf1 <- read_csv("/content/Supermarket_Dataset_1 - Sales Data.csv")

# + colab={"base_uri": "https://localhost:8080/", "height": 629} id="yKLiVYqbSbBn" outputId="3527d419-b8f8-496f-ffa0-df5c64ffe5e3"
#Checking the head and tail of the data 

head(salesdf1)
tail(salesdf1)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} id="pBBY4RX5Sgoc" outputId="8fb68689-6409-4408-ab99-9695585a00a8"
# Lists variables in the dataset
names(salesdf1) 

# + colab={"base_uri": "https://localhost:8080/"} id="-L1UmKhRSoLM" outputId="e82b5f9b-cc25-4bf1-c49c-5b1996d9acc3"
#The rows and columns in the data 
cat("The dataset has ", dim(salesdf1)[1], "rows and ", dim(salesdf1)[2], " columns")

# + colab={"base_uri": "https://localhost:8080/", "height": 589} id="6Au1tmKgTnt2" outputId="84a31f0e-e819-4b48-97ab-0d278cd8e9cb"
#checking the datatypes on the columns 
sapply(salesdf1, class)

# + [markdown] id="0oe2x-QuTEpc"
# ## **Tidying the dataset**

# + id="i-7m0bjAS1P-"
## Importing packages for plotting 

library(tidyr)
library(ggplot2)
library(dplyr)

# + colab={"base_uri": "https://localhost:8080/", "height": 306} id="Pwlbj01yTSzU" outputId="187c70ae-1a0c-4d73-8aa7-c3040cf8906c"
names(salesdf1)<- tolower(names(salesdf1)) # make the column names to lowercase 

names(salesdf1)<- str_replace_all(names(salesdf1), c(" " = "_")) # Replacing the white spaces in the column names 

head(salesdf1)

# + colab={"base_uri": "https://localhost:8080/", "height": 204} id="np_eDMSEThJN" outputId="cf01810e-dfa3-4cc5-855d-f85c43b43286"
# Selecting the numerical data (excluding the categorical variables)



sale1 <- salesdf1[,c(6,7,12,14,15,16)]
head(sale1)


# + colab={"base_uri": "https://localhost:8080/", "height": 437} id="mVpYs7U4U7Lp" outputId="126f2f86-752c-4545-ff00-acea868dc53d"
#Checking for outliers 
boxplot(sale1, col = rainbow(ncol(sale1)))

# + colab={"base_uri": "https://localhost:8080/", "height": 51} id="FPo_DoaeVNfF" outputId="9bf78d3c-8d32-4a39-bdd3-e6ee3f5b6781"
# Listing the outliers 
OutVals = boxplot(sale1, plot=FALSE)$out
OutVals

# + colab={"base_uri": "https://localhost:8080/"} id="tNlId8jTVSyd" outputId="302f9c7e-c952-4d84-a28f-f7718be14b2f"
#Removing outliers 
sales <-salesdf1

sales<- salesdf1[-which(salesdf1$cogs %in% OutVals),]
sales<- salesdf1[-which(salesdf1$gross_income %in% OutVals),]
sales<- salesdf1[-which(salesdf1$total %in% OutVals),]
#check the difference 
print(dim(salesdf1))
print(dim(sales))






# + colab={"base_uri": "https://localhost:8080/", "height": 306} id="17TYT6kqivgi" outputId="d5f7e3e7-170b-45b5-9d56-628384987dd5"
head(sales)

# + colab={"base_uri": "https://localhost:8080/", "height": 68} id="ZXw_5wx4nu3C" outputId="2ce764c5-b88f-4392-b8ee-7a158024cc91"
#checking for duplicates 

table(duplicated(sales))

# + colab={"base_uri": "https://localhost:8080/", "height": 51} id="oDqbUyf_WHt-" outputId="a4a4c60c-a54f-466c-a2b7-b842360342d6"
# Checking the number of missing per column/variable
colSums(is.na(sales)) 

# + [markdown] id="FVgw_vRoWU1N"
# ##**Exploratory Data Analysis** 

# + [markdown] id="4GvWlQigc-BL"
# **Univariate analysis**

# + colab={"base_uri": "https://localhost:8080/", "height": 493} id="o9PPz3J5EUn_" outputId="54a0cc2f-b322-4b84-c115-130aea3641aa"
#summary of the dataset 
#Basic descriptive statistics and frequencies.

summary(salesdf1)   

# + colab={"base_uri": "https://localhost:8080/", "height": 340} id="WQkHnv-3E6u-" outputId="da2fb937-cfa9-48b2-943f-358b34e818ec"
#Descriptive statistics for each column 

desc_stats <- data.frame(
  Min = apply(sale1, 2, min),    # minimum
  Max = apply(sale1, 2, max),     # Maximum
  Med = apply(sale1, 2, median), # median
  Mean = apply(sale1, 2, mean),  # mean
  variance= apply(sale1, 2, var),  #Variance 
  SD = apply(sale1, 2, sd)   # Standard deviation
  ) 


#get the min,max, median, mean , variance and standard deviation of numerical variables 
desc_stats <- round(desc_stats, 1)
print(desc_stats)

#Get the mode 

getmode <- function(v) {
   uniqv <- unique(v)
   uniqv[which.max(tabulate(match(v, uniqv)))]
}


attach(sale1)
 print("The mode of the unit_price column is " )
getmode(unit_price);

 print("The mode of the quantity column is " )
 getmode(quantity); 

print("The mode of the cogs column is " )
 getmode(cogs);


print("The mode of the gross_income column is " )
 getmode(gross_income);

print("The mode of the rating  column is " )
 getmode(rating);
 print("The mode of the total  column is " )
 getmode(total);

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="sG4MeOr0lTQv" outputId="44ff83ec-853d-41c2-9ddf-e3cf39d86e55"
#See the  distribution of numeric variables
hist(sale1$unit_price, col='cadetblue')
hist(sale1$quantity, col='cadetblue')
hist(sale1$cogs, col='cadetblue')
hist(sale1$gross_income, col='cadetblue')
hist(sale1$rating, col='cadetblue')
hist(sale1$total, col='cadetblue')


# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="tHF71zWgdTJR" outputId="ae887d01-f10c-4662-8b5e-082045cd363a"
#See the  distribution of categorical variables 
ggplot(salesdf1,aes(x=toupper(payment)))+geom_bar()+xlab(label = "Modes of Payment")+ylab(label = "Frequency of modes of payment")
ggplot(salesdf1,aes(x=toupper(gender)))+geom_bar()+xlab(label = "Males and Females")+ylab(label = "Frequency of males and females")
ggplot(salesdf1,aes(x=toupper(branch)))+geom_bar()+xlab(label = "Types of branches")+ylab(label = "Frequency of various branches")
ggplot(salesdf1,aes(x=toupper(customer_type)))+geom_bar()+xlab(label = "Customer type")+ylab(label = "Frequency of customers")

# + colab={"base_uri": "https://localhost:8080/", "height": 437} id="WPbiW9PpKuRH" outputId="813c2afe-f7b4-4490-acc1-336891ffd84d"
# Margins area
par(oma=c(0,0,0,0)) # all sides have 3 lines of space
par(mar=c(0,0,0,0) + 0.1)
ggplot(salesdf1,aes(x=toupper(product_line)))+geom_bar()+xlab(label = "Customer type")+ylab(label = "Frequency of customers")

# + [markdown] id="C03L4BCecZ-p"
# **Multivariate analysis**

# + colab={"base_uri": "https://localhost:8080/", "height": 556} id="4w1XkWHYEQoA" outputId="4c6dc381-68bb-48f2-c4bb-952215530b31"
install.packages("GGally")
library(GGally)

ggpairs(sale1)

# + colab={"base_uri": "https://localhost:8080/", "height": 522} id="Jjudx6bGfakC" outputId="795c52ca-9547-4251-e8c9-ba750e5b2957"
M<-cor(sale1) #find the correlation
install.packages("corrplot")
library(corrplot)
# Margins area
par(oma=c(0,0,0,0)) # all sides have 3 lines of space
par(mar=c(0,0,0,0) + 0.1)
corrplot(M, method="number") #Compute and visualize the correlation co-eeficients 

# + [markdown] id="SDMw1rLh141l"
# **Feature selection**

# + id="6yZ2TXd5l1Ho"
#Convert the data to factors and integers 
sales$branch <- (as.integer(as.factor(sales$branch)))
sales$customer_type <- (as.integer(as.factor(sales$customer_type)))
sales$gender <- (as.integer(as.factor(sales$gender)))
sales$product_line <- (as.integer(as.factor(sales$product_line)))
sales$payment <- (as.integer(as.factor(sales$payment)))



# + colab={"base_uri": "https://localhost:8080/", "height": 306} id="onz6S_N5l1Ei" outputId="6a3b303b-9b77-4e13-b544-5e401d6b9fa7"
head(sales)

# + [markdown] id="rEmP_2Hu-kML"
# Feature ranking 

# + colab={"base_uri": "https://localhost:8080/"} id="kqHfhUcO17ii" outputId="3584c324-4656-423c-9f3b-16f167960a33"

# We install and load the required packages
# ---
#
install.packages("FSelector")
library(FSelector)


# + colab={"base_uri": "https://localhost:8080/", "height": 183} id="AhIEH8pY3jIx" outputId="fe1bfcab-efcf-408a-b268-7591cab01c52"
# From the FSelector package, we use the correlation coefficient as a unit of valuation. 
# This would be one of the several algorithms contained 
# in the FSelector package that can be used rank the variables.


Scores <- linear.correlation(total~., sale1)
Scores

# + colab={"base_uri": "https://localhost:8080/", "height": 137} id="eYatg8G3-DFC" outputId="dd43bcb4-e6ff-4442-9eb1-308e2622c332"
# We could also set cutoff as a percentage which would indicate 
# that we would want to work with the percentage of the best variables.
# ---
#
Subset2 <-cutoff.k.percent(Scores, 0.4)
as.data.frame(Subset2)

# + colab={"base_uri": "https://localhost:8080/", "height": 200} id="DTLC44Y83PwT" outputId="1711b483-409d-4759-c26e-6f13054b6000"
# Instead of using the scores for the correlation coefficient, 
# we can use an entropy - based approach as shown below;
# ---
# 
Scores2 <- information.gain(total~., sale1)

# Choosing Variables by cutoffSubset <- cutoff.k(Scores2, 5)
# ---
# 
Subset3 <- cutoff.k(Scores2, 5)
as.data.frame(Subset3)

# + [markdown] id="CkyKRvn4JDif"
# We can see that cogs and gross income explain the most about totals in correlation coefficient and entropy

# + [markdown] id="UTNtlWtGL38u"
# Filter methods

# + id="OeSaLEQGLzhe"
suppressWarnings(
        suppressMessages(if
                         (!require(caret, quietly=TRUE))
                install.packages("caret")))
library(caret)

# + id="zzMwYCsKLzkm"
suppressWarnings(
        suppressMessages(if
                         (!require(corrplot, quietly=TRUE))
                install.packages("corrplot")))
library(corrplot)

# + id="4tdW-vowL3fD"
# Calculating the correlation matrix
# ---
#
df <-sales[,c(2:8,11,12, 14,15)]
correlationMatrix <- cor(df)

# + id="KdIcksRILznh"
# Find attributes that are highly correlated
# ---
#
highlyCorrelated <- findCorrelation(correlationMatrix, cutoff=0.75)

# + colab={"base_uri": "https://localhost:8080/", "height": 51} id="ok1J5vNaLzrD" outputId="dc6f9716-1f2b-4b8d-dd2c-5bdf55b5acb0"
highlyCorrelated
names(df[,highlyCorrelated])


# + id="t8SG4wZQLzuC"
# Removing Redundant Features 
# ---
# 
Dataset2<-df[-highlyCorrelated]



# + colab={"base_uri": "https://localhost:8080/", "height": 437} id="T-pZhJSdLzxW" outputId="6d1d3d2e-96bb-4469-dfd2-a5be97c00b84"
# Performing our graphical comparison
# ---
# 
par(mfrow = c(1, 2))
corrplot(correlationMatrix, order = "hclust")
corrplot(cor(Dataset2), order = "hclust")

# + [markdown] id="2EamdpNeNSt_"
# We are able to remove tax and cogs which were highly correlated. We could use the other variables for more analysis 

StarcoderdataJupyterScriptsDedupFiltered