<jupyter_start><jupyter_text>Clustering of Airbnb Lists Source of data: http://insideairbnb.com/get-the-data.html  Load data into Pandas<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
% matplotlib inline
listings = pd.read_csv("data/Airbnb_listings_20171002.csv")
listings.head(5)
features = ['neighbourhood','room_type','price','minimum_nights','number_of_reviews','reviews_per_month','availability_365','last_review']
lists = listings[features]
lists.info()
lists.loc['last_review_datetime'] = pd.to_datetime(lists['last_review'])
lists.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 8934 entries, 0 to last_review_datetime
Data columns (total 8 columns):
neighbourhood        8933 non-null object
room_type            8933 non-null object
price                8933 non-null object
minimum_nights       8933 non-null object
number_of_reviews    8933 non-null object
reviews_per_month    7123 non-null float64
availability_365     8933 non-null object
last_review          7123 non-null object
dtypes: float64(1), object(7)
memory usage: 628.2+ KB<jupyter_text>When "number_of_reviews" is zero, transform the "reviews_per_month" from null to zero.<jupyter_code>lists['reviews_per_month'] = np.where(lists['number_of_reviews'] == 0, 0, lists['reviews_per_month'])
lists.head(10)<jupyter_output><empty_output><jupyter_text>Use apply function convert availability_365 from the number of days available to the percentage of the whole year that is available<jupyter_code>lists['availability_365_perc'] = lists['availability_365'].apply(lambda x: np.round(x/365, 2))
lists.head(5)<jupyter_output><empty_output><jupyter_text>Data visualization<jupyter_code>import seaborn as sns
sns.set_style("whitegrid")
pricing = listings["price"].values
roomType = listings['room_type'].values
# sns.distplot(pricing)
# sns.countplot(x="price", data=listings, palette="Greens_d")
#PYTEST_VALIDATE_IGNORE_OUTPUT
sns.barplot(x="room_type", y="price", data=listings);
# this is how it works to prevent py.test --nbval mark here as an error: https://github.com/computationalmodelling/nbval/issues/6
#ax = sns.barplot(x="neighbourhood", y="availability_365_perc", data=lists)
lists['neighbourhood'] = lists['neighbourhood'].astype('str') 
# listings.groupby('room_type').size().plot(kind = "bar")
#lists.plot.hist(column = "price", bins = 10)<jupyter_output><empty_output><jupyter_text>One-hot encoding<jupyter_code>lists[['price','minimum_nights','number_of_reviews','availability_365_perc']] = lists[['price','minimum_nights','number_of_reviews','availability_365_perc']].apply(pd.to_numeric,errors='coerce')
drop_columns = ['number_of_reviews','availability_365','last_review']
clean_lists = lists.drop(drop_columns, axis = 1)
clean_lists.columns
final_lists = pd.get_dummies(clean_lists)
final_lists = final_lists.dropna()<jupyter_output><empty_output><jupyter_text>Define target variable<jupyter_code>final_lists['affordability'] = (final_lists['price'] > 150) 
target = final_lists['affordability'].values
target[:2]
target.mean()
list_features_columns = list(final_lists.columns.values)
list_features_columns.remove('affordability')
list_features_columns.remove('price')
list_features = final_lists[list_features_columns]<jupyter_output><empty_output><jupyter_text>Train-test split<jupyter_code>from sklearn.cross_validation import train_test_split
list_features_train, list_features_test, target_train, target_test = train_test_split(list_features, target, test_size = 0.3, random_state = 32 )<jupyter_output><empty_output><jupyter_text>Elbow method for optimal k for clustering<jupyter_code>distortions = []
for i in range(1, 11):
    km = KMeans(n_clusters=i, init='k-means++', n_init=10, max_iter=300, random_state=0)
    km.fit(list_features_train)
    distortions.append(km.inertia_)
    
plt.plot(range(1,11), distortions, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Distortion')
plt.show()<jupyter_output><empty_output><jupyter_text>silhouette plot to test k = 3<jupyter_code>from matplotlib import cm
from sklearn.metrics import silhouette_samples
kmeans = KMeans(n_clusters=3, init='k-means++',n_init=10,max_iter=300, tol=1e-04, random_state=0)
kmeans.fit(list_features_train)
labels = kmeans.labels_

cluster_labels = np.unique(labels)
n_clusters = cluster_labels.shape[0]
silhouette_vals = silhouette_samples(list_features_train, labels, metric='euclidean')

y_ax_lower, y_ax_upper = 0, 0
yticks = []
for i, c in enumerate(cluster_labels):
    c_silhouette_vals = silhouette_vals[labels == c]
    c_silhouette_vals.sort()
    y_ax_upper += len(c_silhouette_vals)
    color = cm.jet(float(i) / n_clusters)
    plt.barh(range(y_ax_lower, y_ax_upper),
    c_silhouette_vals,
    height=1.0,
    edgecolor='none',
    color=color)
    yticks.append((y_ax_lower + y_ax_upper) / 2.)
    y_ax_lower += len(c_silhouette_vals)
    
silhouette_avg = np.mean(silhouette_vals)
plt.axvline(silhouette_avg, color="red", linestyle="--")
plt.yticks(yticks, cluster_labels + 1)
plt.ylabel('Cluster')
plt.xlabel('Silhouette coefficient')
plt.show()<jupyter_output><empty_output><jupyter_text>silhouette plot to test k = 4<jupyter_code>kmeans = KMeans(n_clusters=4, init='k-means++',n_init=10,max_iter=300, tol=1e-04, random_state=0)
kmeans.fit(list_features_train)
labels = kmeans.labels_

cluster_labels = np.unique(labels)
n_clusters = cluster_labels.shape[0]
silhouette_vals = silhouette_samples(list_features_train, labels, metric='euclidean')

y_ax_lower, y_ax_upper = 0, 0
yticks = []
for i, c in enumerate(cluster_labels):
    c_silhouette_vals = silhouette_vals[labels == c]
    c_silhouette_vals.sort()
    y_ax_upper += len(c_silhouette_vals)
    color = cm.jet(float(i) / n_clusters)
    plt.barh(range(y_ax_lower, y_ax_upper),
    c_silhouette_vals,
    height=1.0,
    edgecolor='none',
    color=color)
    yticks.append((y_ax_lower + y_ax_upper) / 2.)
    y_ax_lower += len(c_silhouette_vals)
    
silhouette_avg = np.mean(silhouette_vals)
plt.axvline(silhouette_avg, color="red", linestyle="--")
plt.yticks(yticks, cluster_labels + 1)
plt.ylabel('Cluster')
plt.xlabel('Silhouette coefficient')
plt.show()<jupyter_output><empty_output><jupyter_text>Cluster lists with KMeans, k = 3<jupyter_code>from sklearn.cluster import KMeans
kmeans = KMeans(n_clusters=3, random_state=0)
kmeans.fit(list_features_train)
kmeans.predict(list_features) 
kmeans.cluster_centers_.shape
print ("cluster centers:")
print (kmeans.cluster_centers_)
labels = kmeans.labels_
list_features_train["cluster"] = labels
list(list_features_train)
g = sns.FacetGrid(list_features_train, col="cluster")
g = g.map(plt.scatter, "reviews_per_month", "minimum_nights", marker=".")<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup