# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="Ts_Ae-UdFVTX" colab_type="code" colab={}
import pandas as pd
import numpy as np


# + [markdown] id="3TtoVSXyzdzq" colab_type="text"
# # Combining dataframes
#
#

# + id="qGy0ch2y5MeE" colab_type="code" colab={}
dataFrame1 =  pd.DataFrame({ 'StudentID': [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29], 'Score' : [89, 39, 50, 97, 22, 66, 31, 51, 71, 91, 56, 32, 52, 73, 92]})
dataFrame2 =  pd.DataFrame({'StudentID': [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30], 'Score': [98, 93, 44, 77, 69, 56, 31, 53, 78, 93, 56, 77, 33, 56, 27]})

# In the dataset above, the first column contains information about student identifier and the second column contains their respective scores in any subject. The structure of the dataframes is same in the bothe case. In this case, we would need to concatenate both of them. 

# + id="jkgHbGof7jwO" colab_type="code" outputId="84a6cd36-c185-424a-8312-e2ca607930fe" colab={"base_uri": "https://localhost:8080/", "height": 979}
# We can do that by using Pandas concat() method. 

dataframe = pd.concat([dataFrame1, dataFrame2], ignore_index=True)
dataframe

# + [markdown] id="U8KearUy9X6g" colab_type="text"
# The argument ignore_index creates new index and its absense keeps the original indices. Note, we combined the dataframes along axis=0, that is to say, we combined together along same direction. What if we want to combine both side by side. Then we have to specify axis = 1.  Check the output and see the difference. 

# + id="jaH8srvM-kNo" colab_type="code" outputId="71d847f6-eccf-4902-b34b-3ac2eb6f4a39" colab={"base_uri": "https://localhost:8080/", "height": 514}
pd.concat([dataFrame1, dataFrame2], axis=1)

# + [markdown] id="MzhOAb7BCf33" colab_type="text"
# # Merging
#
# In the first example, you received two files for same subject. Now, consider the use case where you are teaching two courses. So, you will get two dataframes from each sections: two for Software engieering course and another two for Introduction to Machine learning course. Check the figure given below:

# + id="vSlCBOZ4EH-1" colab_type="code" colab={}
df1SE =  pd.DataFrame({ 'StudentID': [9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29], 'ScoreSE' : [22, 66, 31, 51, 71, 91, 56, 32, 52, 73, 92]})
df2SE =  pd.DataFrame({'StudentID': [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30], 'ScoreSE': [98, 93, 44, 77, 69, 56, 31, 53, 78, 93, 56, 77, 33, 56, 27]})

df1ML =  pd.DataFrame({ 'StudentID': [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29], 'ScoreML' : [39, 49, 55, 77, 52, 86, 41, 77, 73, 51, 86, 82, 92, 23, 49]})
df2ML =  pd.DataFrame({'StudentID': [2, 4, 6, 8, 10, 12, 14, 16, 18, 20], 'ScoreML': [93, 44, 78, 97, 87, 89, 39, 43, 88, 78]})

# + [markdown] id="aI4ttuW7EmxY" colab_type="text"
# As you can see in the dataset above, you have two dataframes for each subjects. So the first task would be to concatenate these two subjects into one. Secondly, these students have taken Introduction to Machine Learning course as well. So, we need to merge these score into the same dataframes. There are several ways to do this. Let us explore some options. 

# + id="wNLb0IxdFzde" colab_type="code" outputId="7ef106a3-fcfa-414b-e67f-580a1cf98917" colab={"base_uri": "https://localhost:8080/", "height": 855}
# Option 1
dfSE = pd.concat([df1SE, df2SE], ignore_index=True)
dfML = pd.concat([df1ML, df2ML], ignore_index=True)

df = pd.concat([dfML, dfSE], axis=1)
df

# + id="czCSen-bGY3R" colab_type="code" outputId="4ba02463-5beb-4311-bd07-f661f6a4fd15" colab={"base_uri": "https://localhost:8080/", "height": 700}
# Option 2
dfSE = pd.concat([df1SE, df2SE], ignore_index=True)
dfML = pd.concat([df1ML, df2ML], ignore_index=True)

df = dfSE.merge(dfML, how='inner')
df

# Here, you will perform inner join with each dataframe. That is to say, if an item exists on the both dataframe, will be included in the new dataframe. This means, we will get the list of students who are appearing in both the courses. 

# + id="lHcrG5LwHQTf" colab_type="code" outputId="49234e55-105c-4117-c704-8c1498a72065" colab={"base_uri": "https://localhost:8080/", "height": 855}
# Option 3
dfSE = pd.concat([df1SE, df2SE], ignore_index=True)
dfML = pd.concat([df1ML, df2ML], ignore_index=True)

df = dfSE.merge(dfML, how='left')
df

# + id="mD3vkd-3JNTF" colab_type="code" outputId="955218c2-efaf-4e37-ee30-6618f4a3bddc" colab={"base_uri": "https://localhost:8080/", "height": 824}
# Option 4
dfSE = pd.concat([df1SE, df2SE], ignore_index=True)
dfML = pd.concat([df1ML, df2ML], ignore_index=True)

df = dfSE.merge(dfML, how='right')
df

# + id="XubRsJqMJVIZ" colab_type="code" outputId="0acd9ce3-9e5f-4459-a601-431d8a15196d" colab={"base_uri": "https://localhost:8080/", "height": 979}
# Option 5
dfSE = pd.concat([df1SE, df2SE], ignore_index=True)
dfML = pd.concat([df1ML, df2ML], ignore_index=True)

df = dfSE.merge(dfML, how='outer')
df

# + id="QOZQFDfQPQSH" colab_type="code" outputId="a07f78b6-c01d-403f-bcc1-5586410a074c" colab={"base_uri": "https://localhost:8080/", "height": 359}
df = pd.read_csv('https://raw.githubusercontent.com/PacktPublishing/hands-on-exploratory-data-analysis-with-python/master/Chapter%204/sales.csv')
df.head(10)

# + id="3Z4LekePPX_t" colab_type="code" outputId="77654617-9022-4c46-e4fc-be556e1c3e3a" colab={"base_uri": "https://localhost:8080/", "height": 359}
#@title Default title text
#Add new colum that is the total price based on the quantity and the unit price

df['TotalPrice'] = df['UnitPrice'] * df['Quantity']
df.head(10)

# + id="e0EtWNvlca71" colab_type="code" outputId="623dc411-2948-4f85-f168-ebf2a6ac1678" colab={"base_uri": "https://localhost:8080/", "height": 238}
df['Company'].value_counts()

# + id="xfCdYctmc6o9" colab_type="code" outputId="0dc6124c-9dec-47ea-bf72-b811fdf6d054" colab={"base_uri": "https://localhost:8080/", "height": 297}
df.describe()

# + [markdown] id="wU5_yJnAtImC" colab_type="text"
# ## Reshaping with Hierarchical Indexing

# + id="T3WyVk2JtHzj" colab_type="code" outputId="fd7b98d3-cdb7-48ba-c4b6-aca3448c4e12" colab={"base_uri": "https://localhost:8080/", "height": 142}
data = np.arange(15).reshape((3,5))
indexers = ['Rainfall', 'Humidity', 'Wind']
dframe1 = pd.DataFrame(data, index=indexers, columns=['Bergen', 'Oslo', 'Trondheim', 'Stavanger', 'Kristiansand'])
dframe1

# + id="eJusYeXfvjQj" colab_type="code" outputId="35cb96a5-7083-49d1-9c59-f013b0fea7bb" colab={"base_uri": "https://localhost:8080/", "height": 289}
stacked = dframe1.stack()
stacked

# + id="gbjBsUq1vs_P" colab_type="code" outputId="ebd576b3-e098-4563-b849-52fc279e1440" colab={"base_uri": "https://localhost:8080/", "height": 142}
stacked.unstack()

# + id="cBT2LtcKv5r-" colab_type="code" outputId="a7b68666-61f8-4d52-96b5-5f6d67735403" colab={"base_uri": "https://localhost:8080/", "height": 111}
series1 = pd.Series([000, 111, 222, 333], index=['zeros','ones', 'twos', 'threes'])
series2 = pd.Series([444, 555, 666], index=['fours', 'fives', 'sixs'])

frame2 = pd.concat([series1, series2], keys=['Number1', 'Number2'])
frame2.unstack()

# + [markdown] id="sK21LWxU7_5d" colab_type="text"
# # Data deduplication

# + id="qQ5aZh3R8Cmh" colab_type="code" outputId="93df52b1-23f7-4381-c581-729b87e1777b" colab={"base_uri": "https://localhost:8080/", "height": 266}
frame3 = pd.DataFrame({'column 1': ['Looping'] * 3 + ['Functions'] * 4, 'column 2': [10, 10, 22, 23, 23, 24, 24]})
frame3

# + id="Ptgbp7Lz-YhP" colab_type="code" outputId="e6a79780-fb1b-4125-f03b-215291a2ec68" colab={"base_uri": "https://localhost:8080/", "height": 153}
frame3.duplicated()

# + id="kGZwl8nZ-nAy" colab_type="code" outputId="eb07dc61-d557-4390-f785-91c4aaa3b111" colab={"base_uri": "https://localhost:8080/", "height": 173}
frame4 = frame3.drop_duplicates()
frame4

# + id="Tczy4Zo4-zj1" colab_type="code" outputId="9eb04678-ff7a-48c0-89a2-a1eca062e665" colab={"base_uri": "https://localhost:8080/", "height": 173}
frame3['column 3'] = range(7)
frame5 = frame3.drop_duplicates(['column 2'])
frame5

# + [markdown] id="V_IIMFO8nUOP" colab_type="text"
# # Replacing values

# + id="QAg3Cvs4pbMg" colab_type="code" colab={}
import numpy as np


# + id="tGrxoEnomux6" colab_type="code" outputId="816c0138-9471-4f0d-bba8-ef014a5354d7" colab={"base_uri": "https://localhost:8080/", "height": 328}
replaceFrame = pd.DataFrame({'column 1': [200., 3000., -786., 3000., 234., 444., -786., 332., 3332. ], 'column 2': range(9)})
replaceFrame.replace(to_replace =-786, value= np.nan)


# + id="IgEE2Y6QqS1e" colab_type="code" outputId="0665e4ee-5783-43ed-b19b-ad2384233266" colab={"base_uri": "https://localhost:8080/", "height": 328}
replaceFrame = pd.DataFrame({'column 1': [200., 3000., -786., 3000., 234., 444., -786., 332., 3332. ], 'column 2': range(9)})
replaceFrame.replace(to_replace =[-786, 0], value= [np.nan, 2])


# + [markdown] id="HNY_PNmcmcHg" colab_type="text"
# # Handling missing data

# + id="39Ww1TLcmfS7" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 204} outputId="ed86912f-119b-4d87-dfad-ae637a303027"
data = np.arange(15, 30).reshape(5, 3)
dfx = pd.DataFrame(data, index=['apple', 'banana', 'kiwi', 'grapes', 'mango'], columns=['store1', 'store2', 'store3'])
dfx

# + id="5xIB7OoLnP6d" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="be61e1d6-542d-4836-947f-a2b7abd300f8"
dfx['store4'] = np.nan
dfx.loc['watermelon'] = np.arange(15, 19)
dfx.loc['oranges'] = np.nan
dfx['store5'] = np.nan
dfx['store4']['apple'] = 20.
dfx

# + id="rmmnyON7n5Aw" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="92682fe9-b776-4c27-fb94-87f140edec03"
dfx.isnull()

# + id="cRxkzSDVoRdQ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="343d1a39-d6e7-415b-cf4e-54886ceca871"
dfx.notnull()

# + id="jX7hvqfHn895" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 119} outputId="f27095b4-98ba-430b-b54a-9741d5294efd"
dfx.isnull().sum()

# + id="zOnkTx3QoD_J" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="d0b94ab8-71c2-4222-c6a0-16bc6ff5d6e7"
dfx.isnull().sum().sum()

# + id="Av3zejZKoIV1" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 119} outputId="036ca1dd-e939-49c2-f5eb-0d6674c034cb"
dfx.count()

# + id="iEFQpdEboWzT" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 68} outputId="b73aa208-73ed-416d-fc79-9c1aa58f3c13"
dfx.store4[dfx.store4.notnull()]

# + id="K7t99jEDpd4t" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 68} outputId="7543a2b9-f0e4-487d-b623-7c674034dcb1"
dfx.store4.dropna()


# + id="GcEKe8kgptUp" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 49} outputId="c2a9e8bc-77f1-469b-cbfb-400b1835a322"
dfx.dropna()

# + id="v6FoQukXp14V" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 235} outputId="1325a169-4c64-4981-b0ef-7d43c5d5d68a"
dfx.dropna(how='all')

# + id="_p-FcoMTqBPR" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="35b013f4-6da8-4d07-fa8c-ca2ab787702c"
dfx.dropna(how='all', axis=1)

# + id="OD-aF7lNqRaX" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="90409a08-edfa-4abd-e762-0ba9c2758d84"
dfx2 = dfx.copy()
dfx2.loc['oranges'].store1 = 0
dfx2.loc['oranges'].store3 = 0
dfx2

# + id="MQkyLWCRqj7g" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="81f42642-ae31-418c-91e8-312f89bd82ce"
dfx2.dropna(how='any', axis=1)

# + id="e3t6-Yiuqqz4" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="73972fa2-fe0a-4b38-f166-50fb7f2afb7f"
dfx.dropna(thresh=5, axis=1)


# + [markdown] id="5yPGxdb_rI32" colab_type="text"
# ## NaN values in mathematical operations

# + id="zZvjkIEuq4Z8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="32d50032-43a5-42e3-fc85-85ddee946358"
ar1 = np.array([100, 200, np.nan, 300])
ser1 = pd.Series(ar1)

ar1.mean(), ser1.mean()

# + id="YIwdqe7mrr9w" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="daffb467-f2b1-4907-ada8-3b124c847ab6"
ser2 = dfx.store4
ser2.sum()

# + id="7d3AIK2ZsJ9Q" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="6678a388-9c5c-4bab-b3f4-ec11d3f49416"
ser2.mean()

# + id="xR151fYmsPw9" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 153} outputId="47dad46c-dd36-4556-8c99-77bd7a4f0df2"
ser2.cumsum()

# + id="dLGUzar9sVlp" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 153} outputId="b5d30f48-2a91-4b68-df60-43b077f1764d"
dfx.store4 + 1

# + [markdown] id="1lwepowxsa0Z" colab_type="text"
# ## Filling in missing data
#

# + id="eDxqlyzQscvF" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="a9bee2bc-4c25-4305-902a-be0790349a06"
filledDf = dfx.fillna(0)
filledDf

# + id="c6gD2LamslI1" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 119} outputId="68a0526e-b80e-40b3-e352-9eb4f5a089fd"
dfx.mean()

# + id="WlRGX4sBsz77" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 119} outputId="0f5c387a-dfa2-4678-c471-6d6b10a5a5e2"
filledDf.mean()

# + [markdown] id="zCUfxrM8tHUv" colab_type="text"
# ## Forward and backward filling of the missing values

# + id="UgFKn16ys36G" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 153} outputId="edb9d04a-c5bc-4578-a2c1-665823d944b0"
dfx.store4.fillna(method='ffill')

# + id="Dtadk-EOtSvs" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 153} outputId="b8c96391-4f24-4ff9-8bdc-c88705544af5"
dfx.store4.fillna(method='bfill')

# + [markdown] id="EbC--IUftWKD" colab_type="text"
# ## Filling with index labels
#

# + id="CpJq6vx0tiNu" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 85} outputId="67aa83c2-52db-488c-a2ca-6831954a8d98"
to_fill = pd.Series([14, 23, 12], index=['apple', 'mango', 'oranges'])
to_fill

# + id="dLNp_tiKt7GX" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 153} outputId="5cf066cb-b1b8-4012-933f-47530999f366"
dfx.store4.fillna(to_fill)


# + id="aGyjC6l_uHsV" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 266} outputId="8ec3e24d-1ff0-4aee-e1f4-159a5042f160"
dfx.fillna(dfx.mean())

# + [markdown] id="mdaffGmHuMxx" colab_type="text"
# ## Interpolation of missing values

# + id="llLUU_d1uPoT" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 119} outputId="54be303d-4465-4e69-e6b8-5b35540d9fac"
ser3 = pd.Series([100, np.nan, np.nan, np.nan, 292])
ser3.interpolate()

# + id="HH81nHrXvCbc" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 102} outputId="91e6c734-6816-49f0-9d1e-edfa3736b44f"
from datetime import datetime
ts = pd.Series([10, np.nan, np.nan, 9], 
               index=[datetime(2019, 1,1), 
                      datetime(2019, 2,1), 
                      datetime(2019, 3,1),
                      datetime(2019, 5,1)])

ts

# + id="NYf2f_tOwBDv" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 102} outputId="831a3204-17e8-46db-80cf-4d3428ec27bd"
ts.interpolate()

# + id="3wYxBGUTwaE1" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 102} outputId="f3bf81cb-250f-4588-cc95-5ece560fd8e1"
ts.interpolate(method='time')

# + [markdown] id="JWStq05jt6E-" colab_type="text"
# # Renaming axis indexes

# + id="XclTPUwbt8U2" colab_type="code" outputId="532d9f0e-8d68-420e-a9d1-9ae9ae4bd379" colab={"base_uri": "https://localhost:8080/", "height": 142}
data = np.arange(15).reshape((3,5))
indexers = ['Rainfall', 'Humidity', 'Wind']
dframe1 = pd.DataFrame(data, index=indexers, columns=['Bergen', 'Oslo', 'Trondheim', 'Stavanger', 'Kristiansand'])
dframe1

# + id="O0Qv2t4kuD8x" colab_type="code" outputId="e283cb88-f281-45ae-e751-c0475b98726e" colab={"base_uri": "https://localhost:8080/", "height": 142}
# Say, you want to transform the index terms to capital letter. 
dframe1.index = dframe1.index.map(str.upper)
dframe1

# + id="Bry9nqFduoG3" colab_type="code" outputId="a8b0ab83-dfd9-4254-8a86-a5a8f685e17b" colab={"base_uri": "https://localhost:8080/", "height": 142}
dframe1.rename(index=str.title, columns=str.upper)

# + [markdown] id="Ihbm7F0po4AI" colab_type="text"
# # Discretization and binning
#

# + id="nmXviJEMo7Po" colab_type="code" outputId="5bb892e1-3d7c-49a7-e044-84ed4490d467" colab={"base_uri": "https://localhost:8080/", "height": 68}
import pandas as pd

height =  [120, 122, 125, 127, 121, 123, 137, 131, 161, 145, 141, 132]

bins = [118, 125, 135, 160, 200]

category = pd.cut(height, bins)

category

# + id="TBUhvCRkBzej" colab_type="code" outputId="a3af0c0a-19cb-4942-c666-f90bc673d8c0" colab={"base_uri": "https://localhost:8080/", "height": 102}
pd.value_counts(category)

# + id="_g7NDEETCBgJ" colab_type="code" outputId="752c7e92-19c1-4a1e-d413-0bd9a8861b16" colab={"base_uri": "https://localhost:8080/", "height": 68}
category2 = pd.cut(height, [118, 126, 136, 161, 200], right=False)

category2

# + id="2cz187zECbjJ" colab_type="code" outputId="b1a5303c-93a3-41e3-b725-cd3e5590b326" colab={"base_uri": "https://localhost:8080/", "height": 68}
bin_names = ['Short Height', 'Averge height', 'Good Height', 'Taller']
pd.cut(height, bins, labels=bin_names)

# + id="FGZHymZRDSPf" colab_type="code" outputId="1e39c135-33cc-4c98-aea4-87e4564a711e" colab={"base_uri": "https://localhost:8080/", "height": 85}
# Number of bins as integer
import numpy as np

pd.cut(np.random.rand(40), 5, precision=2)


# + id="T7ujAia3DpXt" colab_type="code" outputId="cee7ac3a-2b7e-4e4a-92ec-dc69950d0eb2" colab={"base_uri": "https://localhost:8080/", "height": 88}
randomNumbers = np.random.rand(2000)
category3 = pd.qcut(randomNumbers, 4) # cut into quartiles
category3

# + id="a-sYRwOYECME" colab_type="code" outputId="3c3c6b76-e93f-4528-b857-551befa8a6e8" colab={"base_uri": "https://localhost:8080/", "height": 102}
pd.value_counts(category3)

# + id="8ZN9C71dSyrz" colab_type="code" outputId="b870424e-9514-4233-b4bc-0284ab4a8d0b" colab={"base_uri": "https://localhost:8080/", "height": 88}
pd.qcut(randomNumbers, [0, 0.3, 0.5, 0.7, 1.0])

# + id="jhUjAjKKb91c" colab_type="code" outputId="88a5243a-380b-4d05-e2e1-e9cfebcd9fd4" colab={"base_uri": "https://localhost:8080/", "height": 359}
df = pd.read_csv('https://raw.githubusercontent.com/PacktPublishing/hands-on-exploratory-data-analysis-with-python/master/Chapter%204/sales.csv')
df.head(10)

# + id="d_CNd0N3b_NE" colab_type="code" outputId="d0203359-9d3e-4c21-e1bf-816ce2c083cb" colab={"base_uri": "https://localhost:8080/", "height": 297}
df.describe()

# + id="jzwTIHn2cGVI" colab_type="code" outputId="1a194815-e0d1-42aa-82d2-654fc2ad6c0f" colab={"base_uri": "https://localhost:8080/", "height": 359}
# Find values in order that exceeded 
df['TotalPrice'] = df['UnitPrice'] * df['Quantity']
df.head(10)

# + id="M1W2NgpKcdIV" colab_type="code" outputId="9692c64f-c5ba-4475-c2f1-6615a6f4719b" colab={"base_uri": "https://localhost:8080/", "height": 221}
# Find transaction exceeded 3000000
TotalTransaction = df["TotalPrice"]
TotalTransaction[np.abs(TotalTransaction) > 3000000]

# + id="DVCsCQoVewpn" colab_type="code" outputId="cf4e7483-9dbb-4f10-de45-57fe5a42c5b6" colab={"base_uri": "https://localhost:8080/", "height": 328}
df[np.abs(TotalTransaction) > 6741112]

# + [markdown] id="CrGqRDtiIc1v" colab_type="text"
# # Permunation and Random sampling

# + id="EcP-eneGIb-g" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 359} outputId="95e49583-7c30-417a-af84-80c536f894bc"
dat = np.arange(80).reshape(10,8)
df = pd.DataFrame(dat)

df

# + id="KhcFpNm8JGqY" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="200e9d84-f898-470d-f2b5-fc862159ec8d"
sampler = np.random.permutation(10)
sampler

# + id="oA1ZWkOxJH_6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 359} outputId="9fada503-d996-4b5a-d1c7-c217341b3b0f"
df.take(sampler)

# + id="99md1Z4GJQj9" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 142} outputId="b144e870-1821-4d23-aae4-78c03a1aeb19"
# Random sample without replacement

df.take(np.random.permutation(len(df))[:3])

# + id="TTMIv3DFJd68" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="8d56dadd-19d5-46e3-89ce-269e668fcc6b"
# Random sample with replacement
sack = np.array([4, 8, -2, 7, 5])
sampler = np.random.randint(0, len(sack), size = 10)
sampler

# + id="sz1O1ITkJ0w2" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="18e6fb1e-d370-4c42-f822-11edcb5d85e8"
draw = sack.take(sampler)
draw

# + [markdown] id="GCJ5KRuvWW41" colab_type="text"
# # Dummy variables

# + id="aAPhGPlHWbAw" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 235} outputId="f763c6ff-5dad-49f1-b2cf-4ff986bbf16c"
df = pd.DataFrame({'gender': ['female', 'female', 'male', 'unknown', 'male', 'female'], 'votes': range(6, 12, 1)})
df

# + id="mBqTJ0EWYGzS" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 235} outputId="c0dd864c-ac1b-45a4-a116-00bf132f550e"
pd.get_dummies(df['gender'])

# + id="6bTNv3jqYTR5" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 235} outputId="4acff651-22c0-4144-f8d1-d2b0dd79bd84"
dummies = pd.get_dummies(df['gender'], prefix='gender')
dummies

# + id="fTDLBPohYUaF" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 235} outputId="aba27eda-5499-4733-8efb-72702edf86ed"
with_dummy = df[['votes']].join(dummies)
with_dummy

StarcoderdataJupyterScriptsDedupFiltered