# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # How Does a Bike-Share Navigate Speedy Success
#

# ![](ghtop_images/header2.png)

# ## Introduction
#
# Some quick analysis on Cycle bike-share data from Chicago
#
# <i>Welcome to the Cyclistic bike-share analysis case study! In this case study, you will perform many real-world tasks of a junior
# data analyst. You will work for a fictional company, Cyclistic, and meet different characters and team members. In order to
# answer the key business questions, you will follow the steps of the data analysis process: ask, prepare, process, analyze,
# share, and act. Along the way, the Case Study Roadmap tables — including guiding questions and key tasks — will help you
# stay on the right path.
# By the end of this lesson, you will have a portfolio-ready case study. Download the packet and reference the details of this
# case study anytime. Then, when you begin your job hunt, your case study will be a tangible way to demonstrate your
# knowledge and skills to potential employers.</i>

# ### Some imports

# + deletable=false editable=false run_control={"frozen": true}
# import pandas as pd
# from bs4 import BeautifulSoup
# import numpy as np
# import requests
# # from datetime import datetime, timezone
# import os
# import geopandas as gpd
#
# #get the current working directory
# owd=os.getcwd()
# -

# ## A function to extract web zip files and save to /dat folder
#
# ### We'll just look at Q1 for 2020 data
#
# The data for 2020 is split into Q1 and the rest is for each month. So we need to download and convert each to a data frame and then combine them together

# + deletable=false editable=false run_control={"frozen": true}
# def extractStuff(url):
#     import requests, zipfile, io
#     import os
#     owd=os.getcwd()
#     r = requests.get(url)
#     z = zipfile.ZipFile(io.BytesIO(r.content))
#     z.extractall(owd+"/dat/")
# URL="https://divvy-tripdata.s3.amazonaws.com/"
# noma =[['202004-divvy-tripdata'],
# ['202005-divvy-tripdata'],
# ['202006-divvy-tripdata'],
# ['202007-divvy-tripdata'],
# ['202008-divvy-tripdata'],
# ['202009-divvy-tripdata'],
# ['202010-divvy-tripdata'],
# ['202011-divvy-tripdata'],
# ['202012-divvy-tripdata']]
#     

# + deletable=false editable=false run_control={"frozen": true}
# for nom in noma:
#     extractStuff(URL+ nom[0]+".zip")
# -

# ### Load files, put in pandas data frame and have a look
#
# #### Lets load each csv and combine them

# + deletable=false editable=false run_control={"frozen": true}
# for i,nom in enumerate(noma):
#     if i>0:
#         df=pd.read_csv(owd+"/dat/"+nom[0]+'.csv')
#         dfAll=pd.concat([df,dfAll])    
#         print('1 ',nom[0])
#     else:
#         dfAll=pd.read_csv(owd+"/dat/"+nom[0]+'.csv')
#         print('2',nom[0])
# -

# ### How many NaN stations?

# + deletable=false editable=false run_control={"frozen": true}
# print('The percentage start stations NaN = {}'.format(100*np.shape(dfAll[dfAll['start_station_id'].isna()])[0] / np.shape(dfAll)[0]) )#95 282 3 114 796
#
# print('The percentage end stations NaN = {}'.format(100*np.shape(dfAll[dfAll['end_station_id'].isna()])[0] / np.shape(dfAll)[0]) )#95 282 3 114 796
#
# bothNa=dfAll[dfAll['start_station_id'].isna() | dfAll['end_station_id'].isna()] 
# print('The percentage start stations NaN = {}'.format(100*np.shape(bothNa)[0] / np.shape(dfAll)[0]) )#95 282 3 114 796

# + deletable=false editable=false run_control={"frozen": true}
# import copy
# dfUse=copy.copy(dfAll[dfAll['start_station_id'].notnull() & dfAll['end_station_id'].notnull()])
# dfUse.describe(include='all')
# -

# ### Now we need to convert the dates from object (i.e. string) to date format
#
# #### next add a new column as time for hire in hours

# + deletable=false editable=false run_control={"frozen": true}
# dfUse.loc[:,'started_at']=pd.to_datetime(dfUse['started_at'],infer_datetime_format=True)
# dfUse.loc[:,'ended_at']=pd.to_datetime(dfUse['ended_at'],infer_datetime_format=True)
#
# delta=dfUse.iloc[:,3]-dfUse.iloc[:,2]
# dd=delta.dt.total_seconds()/(60*60)
# dfUse.insert(2,"hire_time_h",dd)
# dfUse.head()
# -

# #### Maybe we want the day of the week?
# The day of the week with Monday=0, Sunday=6.

# + deletable=false editable=false run_control={"frozen": true}
# dfUse.insert(3,'day_week',dfUse.loc[:,'started_at'].dt.dayofweek)
# dfUse.head()
# -

# ### Lets also get the time on its own

# + deletable=false editable=false run_control={"frozen": true}
# dfUse.insert(4,'time_day',dfUse.loc[:,'started_at'].dt.hour + dfUse.loc[:,'started_at'].dt.minute/60)
# dfUse.head()
# -

# ### And the distance travelled

# + deletable=false editable=false run_control={"frozen": true}
# def distanceLatLong(lat1,lon1,lat2,lon2):
#     
#     import numpy as np
#
#     def deg2rad(deg):
#         return deg * np.pi/180
#
#     R = 6371; # Radius of the earth in km
#     dLat = deg2rad(lat2-lat1)  # deg2rad below
#     dLon = deg2rad(lon2-lon1)
#     a = np.sin(dLat/2) * np.sin(dLat/2) + \
#     np.cos(deg2rad(lat1)) * np.cos(deg2rad(lat2)) * \
#     np.sin(dLon/2) * np.sin(dLon/2)
#
#     c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
#     d = R * c ## Distance in km
#  
#     return d

# + deletable=false editable=false run_control={"frozen": true}
# d=distanceLatLong(dfUse["start_lat"].values,dfUse["start_lng"].values,dfUse["end_lat"].values,dfUse["end_lng"].values)
# dfUse.insert(11,'distance',d)
# dfUse.describe()
# -

# ### Some issues arose above
# - hire_time_h max and min values
#
# Looks like the error is there from the start, so lets delete them
#
# do the same for long times

# + deletable=false editable=false run_control={"frozen": true}
# dfUse=dfUse[dfUse.hire_time_h>=0]
#
# dfUse=dfUse[dfUse.hire_time_h<24]
# -

# ### Lets drop some columns for space

# + deletable=false editable=false run_control={"frozen": true}
# dfUse.drop(columns=["ride_id", "started_at","ended_at","start_station_name","end_station_name"],inplace=True)
# #,"start_station_id","end_station_id"]
# -

# ### save

# + deletable=false editable=false run_control={"frozen": true}
# dfUse.to_csv('/data/df_2020.csv')
# df=copy.copy(dfUse)
# -

df = pd.read_csv('/data/df_2020.csv')
df

# ### Put frequency location onto a map

# + deletable=false editable=false run_control={"frozen": true}
# import folium
# from folium import plugins
# from folium.plugins import HeatMap
#
# lat=df['start_lat'].values
# lon=df['start_lng'].values
# latlon = [lat, lon]
#
# maps = folium.Map(location=[lat[0],lon[0]],
#                     zoom_start = 11)
#
# latlon=np.transpose(latlon)
#
# # Plot it on the map
# HeatMap(latlon).add_to(maps)
#
# # Display the map
# maps
# -

# ![](ghtop_images/ChiHeat.png)

# Bit of a mess, grouping by region may be better

# ## Plot some choroplots
# open and modify the geojson file- seems to make life easier later

# + deletable=false editable=false run_control={"frozen": true}
# import geopandas as gpd
# fname='Chicago.geojson'
# chicago = gpd.read_file(fname)
#
# # neighborhoods aren't unique so lets use the index and call it ID
# chicago.reset_index(inplace=True)
# chicago.rename(columns={'index':'ID'},inplace=True)
#
# chicago.drop(columns=['sec_neigh','shape_area','shape_len'],inplace=True)
# chicago.to_file("Chi_.json", driver="GeoJSON")
# chicago.head()
# -

# ### Now we want to convert each station to a region within the json file
#
# #### First let's create a variable for each station, with location and station_id

# + deletable=false editable=false run_control={"frozen": true}
# #use mean here just in case some slight differences- big ones lets hope not!
# dfStat=df.groupby(by=['start_station_id']).mean()
#
# dfStat=dfStat.drop(columns=['Unnamed: 0','hire_time_h','day_week','time_day','distance','end_lat','end_lng'])
# dfStat.reset_index(inplace=True)
# dfStat
# -

# #### Now for each station we want a JSON-area code
# This is *slightly* convoluted
# - scroll through each station
# - for each station find if it's inside a Chigao_JSON region
# - if not we assign if a value 1000
# - for those with no region find the nearest station that has a JSON-region (done in tab after this)
#
# whatChoro = json ID
# whatwhat = station ID

# + deletable=false editable=false run_control={"frozen": true}
# from shapely.geometry import shape, Point
#
#
# whatChoro=[]
# whatwhat=[]
# # check each polygon to see if it contains the point
# i=0
# # scroll through each station
# for istat in range(np.shape(dfStat)[0]):
#     i=0
#     #create a point for the station
#     point=Point(dfStat.loc[istat,'start_lng'],dfStat.loc[istat,'start_lat'])
#     
#     #scroll through each geometery
#     for feature in chicago.ID:
#         polygon = shape(chicago.loc[i,'geometry'])
#         if polygon.contains(point):
#             #this gives the json region ID
#             whatChoro.append(chicago.loc[i,'ID'])
#             #this give the station id
#             whatwhat.append(dfStat.loc[istat,'start_station_id'])
#             break 
#         
#         #if we don't get a match!!
#         if feature==chicago['ID'].iloc[-1]:
#             
#             import copy
#             # find distances lat2/lng2 this location
#             # lat1/lng1 all locations
#             lat1_=copy.copy(dfStat['start_lat'])
#             lon1_=copy.copy(dfStat['start_lng'])
#             lat2_=dfStat.loc[istat,'start_lat']
#             lon2_=dfStat.loc[istat,'start_lng']
#
#             
#             #this gives the json region ID
#             whatChoro.append(1000)
#             #this give the station id
#             whatwhat.append(dfStat.loc[istat,'start_station_id'])
#             
#             
#         i=i+1
# -

# #### This does the cleaning up if they don't have a json id
#
# This will handle when we don't get a match---> a reuse of the distance function with slight mods

# + deletable=false editable=false run_control={"frozen": true}
# def distanceLatLong_v2(lat1_,lon1_,lat2_,lon2_):
#     
#     import numpy as np
#     import math
#     def deg2rad(deg):
#         return deg * np.pi/180
#     def inner(lat1,lon1,lat2,lon2):
#         R = 6371; # Radius of the earth in km
#         dLat = deg2rad(lat2-lat1)  # deg2rad below
#         dLon = deg2rad(lon2-lon1)
#         a = np.sin(dLat/2) * np.sin(dLat/2) + \
#         np.cos(deg2rad(lat1)) * np.cos(deg2rad(lat2)) * \
#         np.sin(dLon/2) * np.sin(dLon/2)
#
#         c = 2 * math.atan2(np.sqrt(a), np.sqrt(1-a))
#         d = R * c ## Distance in km
#         if d==0:
#             d=1000
#         return d
#     if np.shape(lat1_)[0]>1:
#         d=[]
#         for i in range(np.shape(lat1_)[0]):
#             d.append(inner(lat1_[i],lon1_[i],lat2_,lon2_))
#         
#     else:
#         d=inner(lat1_,lon1_,lat2_,lon2_)   
#     
#     return d
# -

# this scrolls through ones we didn't match and finds nearest JSON-id we did match

# + deletable=false editable=false run_control={"frozen": true}
# for i in range(np.shape(dfStat)[0]):
#     if whatChoro[i]==1000:
#              
#         #find distances lat2/lng2 this location
#         # lat1/lng1 all locations
#         lat1_=copy.copy(dfStat['start_lat'])
#         lon1_=copy.copy(dfStat['start_lng'])
#         lat2_=dfStat.loc[i,'start_lat']
#         lon2_=dfStat.loc[i,'start_lng']
#         ind=[idx for idx, element in enumerate(whatChoro) if element==1000]
#         lat1_[ind]=0
#         lon1_[ind]=0
#         d=distanceLatLong_v2(lat1_,lon1_,lat2_,lon2_)
#         indamin=d.index(min(d))
# #         whatwhat[i]=whatwhat[indamin]
#         whatChoro[i]=whatChoro[indamin]
#         
#         print(i,indamin,whatChoro[i],whatChoro[indamin],min(d))
# -

# ### Now we can insert a new column in df with the json ID

# + deletable=false editable=false run_control={"frozen": true}
# chicID=[]
# for stat in df['start_station_id']:
# #     chicID.append(stat)
#     ind=[idx for idx, element in enumerate(whatwhat) if element==stat]
#     try:
#         chicID.append(whatChoro[ind[0]])
#     except:
#         continue

# + deletable=false editable=false run_control={"frozen": true}
# df.insert(0,'ID',chicID)
# df.head()
# -

# ### And represent each JSON region by how many times they're used
# We'll take the count and divide it by the total- and because of the distribution we'll also take the log- basically hires are highly focussed on a few regions with many having low % 

# + deletable=false editable=false run_control={"frozen": true}
# dfG=df.groupby('ID').count()
# dfG.reset_index(inplace=True)
# dfG=dfG[['ID','rideable_type']]
# dfG.rename(columns={'rideable_type':'Frequency'})
# dfG.rideable_type=np.log(dfG.rideable_type/sum(dfG.rideable_type))
# dfG.head()

# + deletable=false editable=false run_control={"frozen": true}
# df.to_csv('/data/dfChoro_2020.csv')
# -

df = pd.read_csv('/data/dfChoro_2020.csv')


# ### Now the plotting
# We first read in the json file, add the df with our frequency values to it then we can plot the data
#

# ### Lets put this in a function to look at differences

def bigChoro(dfIN,colname,choi):
    import folium 
    
    LEGNOM=colname
    if choi=='count':
        dfG=dfIN.groupby('ID').count()
        dfG[colname]=dfG[colname]/(100*274/7)
        myscale = (dfG[colname].quantile((0,0.25,0.5,0.75,0.9,0.95,.97,1))).tolist()
#         np.linspace(dfG[colname].min(),dfG[colname].max(),10)
        LEGNOM='Number of journeys 100s per week'
    elif choi=='mean':
        dfG=dfIN.groupby('ID').mean()
        myscale = np.linspace(dfG[colname].min(),dfG[colname].max(),10)
    elif choi=='sum':
        dfG=dfIN.groupby('ID').sum()
        myscale = np.linspace(dfG[colname].min(),dfG[colname].max(),10)
    elif choi=='dayofweek':
        dfIN=dfIN[['ID',colname]]
        dfG=dfIN.groupby(['ID']).agg(lambda x:x.value_counts().index[0])
        dfG[dfG[colname]>4]=5-dfG[dfG[colname]>4]
        myscale = np.array([-2.,0.,1.,2.,3.,4.])
        LEGNOM='Day of week (-2 to -1 weekend, 0-4 Monday to Friday)'
    elif choi=='mode':
        dfIN=dfIN[['ID',colname]]
        dfIN[colname].astype('int32')
        dfG=dfIN.groupby(['ID']).agg(lambda x:x.value_counts().index[0])
        myscale = np.linspace(dfG[colname].min(),dfG[colname].max(),10)
        
    dfG.reset_index(inplace=True)
    dfG=dfG[['ID',colname]]


    nil=gpd.read_file("Chi_.json")
    nil=nil[['ID','geometry']]

    # merge data frames
    nilpop=nil.merge(dfG,on="ID")

    #initial map
    m = folium.Map(location=[41.884,-87.6247], zoom_start=10,\
       control_scale=True,tiles="Stamen Toner")#,tiles = t_list[1])

    folium.TileLayer('CartoDB positron',name="Light Map",control=False).add_to(m)

   
    # (dfG['rideable_type'].quantile((0,.02,0.1,.25,0.5,0.75,0.9,0.95,0.98,1))).tolist()
    choropleth =folium.Choropleth(
        geo_data="Chi_.json", 
        data=nilpop,
        threshold_scale=myscale,
        columns=['ID',colname],
        name='choropleth',
        fill_color='BuPu',#PuBuGn YlGn PuBuGn YlGnBu RdYlBu
        key_on= "feature.properties.ID",
        fill_opacity=0.7,
        line_opacity=0.2,
        nan_fill_color='gray',
        legend_name=LEGNOM, 
        nan_fill_opacity =.5,
        ).add_to(m)
    folium.LayerControl().add_to(m)
    choropleth.geojson.add_child(
        folium.features.GeoJsonTooltip(['pri_neigh'],labels=False)
        )
    return m

dfIN.columns

# + deletable=false editable=false run_control={"frozen": true}
# colname='distance'#end_station_id
# dfIN= df[df.member_casual=='member']
# m=bigChoro(dfIN,colname,'mean')
# m
# -

# ![](ghtop_images/ChiDist.png)

# + deletable=false editable=false run_control={"frozen": true}
# colname='day_week'#end_station_id
# dfIN= df[df.member_casual=='member']
# m=bigChoro(dfIN,colname,'dayofweek')
# m
# -

# ![](ghtop_images/ChiDay.png)

# + deletable=false editable=false run_control={"frozen": true}
# colname='time_day'#end_station_id
# dfIN= df[df.member_casual=='member']
# m=bigChoro(dfIN,colname,'mode')
# m
# -

# ![](ghtop_images/ChiDayTime.png)

colname='time_day'#end_station_id
dfIN= df[df.member_casual=='member']
m=bigChoro(dfIN,colname,'count')
m

# ![](ghtop_images/ChiCountMember.png)

# + deletable=false editable=false run_control={"frozen": true}
# colname='time_day'#end_station_id
# dfIN= df[df.member_casual=='casual']
# m=bigChoro(dfIN,colname,'count')
# m
# -

# ![](ghtop_images/ChiCountCasual.png)

np.linspace(-2,4,7)
np.array([-2.,0.,1.,2.,3.,4.])

import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt

# +
sns.set_theme(style="ticks")

f, ax = plt.subplots(figsize=(7, 5))
sns.despine(f)

# Draw a nested boxplot to show bills by day and time
sns.histplot(df,hue="member_casual", x="time_day",
            multiple="stack",
            palette="dark:b_r",
            edgecolor=".3",
            linewidth=.5,)
sns.despine(offset=10, trim=True)

# +
from matplotlib.ticker import PercentFormatter
df_=df[df.hire_time_h<5]
df_=df_[df_.hire_time_h>0]
sns.set_theme(style="ticks")

binwidth = 5

f, ax1 = plt.subplots(figsize=(10, 7))
sns.despine(f)
df2_=df_[df_.member_casual=='member']
# Draw a nested boxplot to show bills by day and time
sns.histplot(df2_, x="hire_time_h",
            multiple="stack",
            palette="light:m_r",
            edgecolor=".3",
            linewidth=.5,
            stat='probability',
            log_scale=True,
            ax=ax1,
            label='member')

ax2=ax1.twinx()

df2_=df_[df_.member_casual=='casual']
# Draw a nested boxplot to show bills by day and time
sns.histplot(df2_, x="hire_time_h",
             element="step",fill=False,
             color='red',
            linewidth=.8,
            stat='probability',
            log_scale=True,
            ax=ax2,
            label='casual')

ax1.legend(loc='upper left')
ax2.legend(loc='upper right')
sns.despine(offset=10, trim=True)
# ax.legend('Member','Casual')

# +
sns.set_theme(style="ticks")

f, ax = plt.subplots(figsize=(10, 8))
sns.despine(f)
df_=df[ df["distance"]>0.1 ]
df_= df_[df_["distance"]<20]
df_=df_[df_.member_casual=='member']
# Draw a nested boxplot to show bills by day and time
sns.histplot(data=df_, x="distance",
            edgecolor=".3",
            linewidth=.5,
            stat='probability',
             label='member')
sns.despine(offset=10, trim=True)
df_=df[df["distance"]>0.1]
df_= df_[df_["distance"]<20]
df_=df_[df_.member_casual=='casual']
sns.histplot(data=df_, x="distance",
            linewidth=.8,
            color='r',
            stat='probability',
             label='casual',
            fill=False,
            element='step')

ax.legend(loc='upper right')


# +
# f, ax = plt.subplots(figsize=(7, 5))
sns.set_theme(style="whitegrid")

# Draw a nested boxplot to show bills by day and time
ax=sns.histplot(df,hue="member_casual", 
             x="day_week",palette="dark:b_r",
             multiple="dodge",
             bins=[0 ,1 ,2 ,3, 4, 5, 6,7],
             shrink=.9
            )
sns.despine(offset=20, trim=True)
aa=np.array([0,1,2,3,4,5,6])+.5
ax.set_xticks(aa)
ax.set_xlim([0, 7.5])
lab=['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
ax.set_xticklabels(lab,rotation='vertical')
ax.set_xlabel('Day of the week')

# -

# ### So after a quick look at the data (*maybe some plots need mods) some clear trends:
# - Casuals use bikes more on weekends, members more on weekdays
# - Members tend to use bikes in commuting times 7-9 am and 4-7 pm. Whereas casuals more spread but focussed later
# - Casuals tend to use the bikes for longer and travel further from initial location

StarcoderdataJupyterScriptsDedupFiltered