# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# ##Data Description and Exploration
#
# In this Module, we'll cover
#
# - describe
# - groupby
# - split-apply-combine
# - cross tabs and pivot tables
# - plotting
#

# +
import pandas as pd
import numpy as np

# %matplotlib inline

# -

# Let's get started by reading in the data. 

df = pd.read_csv("/Users/matthewgee/Buildings_Violations_sample_50000_clean.csv")

df.describe()

pd.crosstab(df.violation_status,df.department_bureau, margins=True)

# ##Groupby
#
# Often we want to examine difference among groups based on categorical values. For this `groupby` is incredibly valuable

df[['department_bureau','longitude','latitude']].groupby("department_bureau").mean()

# ##Groupby and Plotting!
#  
# Often we wan to view an aggregation we've generated. How dow we plot groupby objects?

df.columns

df[['violation_date','id']].groupby(['violation_date']).count().head()

grouped = df[['violation_date','id']].groupby(['violation_date']).count()

grouped.plot(figsize=(15,5))

# Notice the dates. Something looks suspicious. Have we converted them to datetime?

df.dtypes

# NOOOO!!!!
#
# Let's fix that.

df.violation_date = pd.to_datetime(df.violation_date)
df.violation_status_date = pd.to_datetime(df.violation_status_date)
df.violation_last_modified_date = pd.to_datetime(df.violation_last_modified_date)

grouped = df[['violation_date','id']].groupby(['violation_date']).count()

grouped.plot(figsize=(15,5))

# Now let's narrow the window that we're looking at. It turns out that having a datetime index to the series can be incredibly helpful.

grouped['2011-01']

grouped['2011-01'].plot(figsize=(15,5))

grouped['2011':'2012'].plot(figsize=(15,5))

# Now let's try to create a multi-line plot. First we need to do a multi-level groupby.

df1 = df[df['violation_status']=='OPEN']
grouped_open = df1[['violation_date','violation_status','inspection_category','id']].groupby(['violation_date','violation_status','inspection_category']).count()
grouped_open.head()

# Now we want to pivot this table so that there is a column for each inspection category. I find this a lot easier to do in pandas when what I'm pivoting is a column rather than an index, so let's reindex

grouped_open.reset_index(inplace=True)
grouped_open.head()

grouped_open_categ = grouped_open.pivot_table('id','violation_date','inspection_category')
grouped_open_categ.head()

grouped_open_categ.plot(figsize=(15,5))

grouped_open_categ_reverse = pd.melt(grouped_open_categ, id_vars=['violation_date'], value_vars=['COMPLAINT','PERIODIC','PERMIT','REGISTRATION'])

pd.value_counts(grouped_open_categ_reverse.inspection_category)

# Let's say we wanted to go ahead and aggregate counts by week or month, not just by day. We can use the handy resample command.

avg_monthly = grouped.resample('M', how='mean')
avg_monthly.plot()

total_monthly = grouped.resample('M', how='sum')
total_monthly.plot()

StarcoderdataJupyterScriptsDedupFiltered