<jupyter_start><jupyter_text>Import libraries:<jupyter_code>import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Creating a dataframe:<jupyter_code>courses = ["Tableau", "GSS", "SQL", "Python"]
dfcourses = pd.DataFrame(data=courses)
dfcourses<jupyter_output><empty_output><jupyter_text>Adding  a new column to dataframe:<jupyter_code>popularity = [3, 4, 2, 1]
dfcourses[1] = popularity
dfcourses<jupyter_output><empty_output><jupyter_text>Renaming column:<jupyter_code>dfcourses.columns = ["Courses", "Popularity"]
dfcourses<jupyter_output><empty_output><jupyter_text>Creating a dataframe from dictionary<jupyter_code>df_from_dict = pd.DataFrame({'Courses' :['Tableau' , 'GSS' , 'SQL' , 'Python'] , "Popularity" : [3,4,
2,1]})
df_from_dict<jupyter_output><empty_output><jupyter_text>Creating date ranges with defining "start - end"<jupyter_code>holiday1=pd.date_range(start="2020-08-16", end="2020-08-32")  # !!!ATTENTION: day must be in range for month
holiday1
holiday1=pd.date_range(start="2020-08-17", end="2020-08-23")  # !!!ATTENTION: day must be in range for month
holiday1<jupyter_output><empty_output><jupyter_text>Creating data ranges with defining "periods"<jupyter_code>holiday2=pd.date_range("2020-08-31", periods=7)
holiday2<jupyter_output><empty_output><jupyter_text>Creating random values to build a df<jupyter_code>random_values1=np.random.random((7,7))
random_values1<jupyter_output><empty_output><jupyter_text>Creating a df from data ranges and random_values<jupyter_code>df_new= pd.DataFrame(random_values1, index=holiday1)
df_new<jupyter_output><empty_output><jupyter_text>Renaming columns<jupyter_code>df_new.columns=["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"
]
df_new<jupyter_output><empty_output><jupyter_text>Showing index values<jupyter_code>df_new.index<jupyter_output><empty_output><jupyter_text>Showing columns<jupyter_code>df_new.columns<jupyter_output><empty_output><jupyter_text>Data types of columns<jupyter_code>df_new.dtypes<jupyter_output><empty_output><jupyter_text>General view of the df<jupyter_code>df_new.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 7 entries, 2020-08-17 to 2020-08-23
Freq: D
Data columns (total 7 columns):
 #   Column     Non-Null Count  Dtype  
---  ------     --------------  -----  
 0   Monday     7 non-null      float64
 1   Tuesday    7 non-null      float64
 2   Wednesday  7 non-null      float64
 3   Thursday   7 non-null      float64
 4   Friday     7 non-null      float64
 5   Saturday   7 non-null      float64
 6   Sunday     7 non-null      float64
dtypes: float64(7)
memory usage: 448.0 bytes<jupyter_text>Shape<jupyter_code>df_new.shape
#Sorting df

df_new.sort_values(by="Monday") #default is ascending=True
df_new.sort_values(by="Monday", ascending=False) #default is ascending=True<jupyter_output><empty_output><jupyter_text>Deleting a column-1<jupyter_code>dfcourses["Prix"]=["100","75","200","300"]
dfcourses
dfcourses.drop(["Popularity"], axis=1, inplace=False)<jupyter_output><empty_output><jupyter_text>Deleting a column-2<jupyter_code>dfcourses.drop(columns=["Popularity"], inplace=False)<jupyter_output><empty_output><jupyter_text>Deleting multiple columns<jupyter_code>dfcourses.drop(columns=["Popularity", "Prix"], inplace=False)<jupyter_output><empty_output><jupyter_text>Deleting a row<jupyter_code>dfcourses.drop([1], axis=0, inplace=False)<jupyter_output><empty_output><jupyter_text>Deleting multiple rows<jupyter_code>dfcourses.drop([2,3], axis=0, inplace=False)<jupyter_output><empty_output><jupyter_text>Deleting a row-Version_2<jupyter_code>dfcourses.drop(dfcourses.index[2], inplace=False)<jupyter_output><empty_output><jupyter_text>Deleting a row based on a column value<jupyter_code>dfcourses.drop(dfcourses.index[dfcourses.Courses=="Tableau"].tolist(), axis=0, inplace=False)
dfcourses=dfcourses.drop(dfcourses[dfcourses.Popularity==4].index)
dfcourses<jupyter_output><empty_output><jupyter_text>Keeping rows based on values in a column using isin()<jupyter_code>dfcourses=dfcourses[dfcourses.Popularity.isin([1,2])]
dfcourses
dfstd = pd.DataFrame({ 'Student_id' :[101,102,103,104] ,
'first_name' :['Ali' , 'Veli' , 'Emre' , 'Hasan'] ,
'country' : ['Turkey' , 'US','UK' , 'France']}, index=[1,2,3,4])
dfstd<jupyter_output><empty_output><jupyter_text>Selecting data using loc--index values<jupyter_code>dfstd.loc[2]
dfstd.loc[2:3]<jupyter_output><empty_output><jupyter_text>Selecting data using iloc<jupyter_code>dfstd.iloc[1]
dfstd.iloc[2:4]<jupyter_output><empty_output><jupyter_text>Conditional Selection<jupyter_code>dfstd.loc[dfstd.Student_id > 102]
dfstd2=dfstd.rename(index={1: 'A', 2:"B", 3:"C", 4:"D"})
dfstd2<jupyter_output><empty_output><jupyter_text>Label based selections<jupyter_code>dfstd2.loc["D"]
dfstd2.loc[["A","D"]]<jupyter_output><empty_output><jupyter_text>Row label selections<jupyter_code>dfstd2[1:3]
dfstd2.loc[:,["Student_id", "country"]]
dfstd2.loc["A":"C",["Student_id", "country"]]<jupyter_output><empty_output><jupyter_text>Data selection using position<jupyter_code>dfstd2.iloc[1][2]
dfstd2.iloc[1:3,:]
dfstd2.iloc[1,:]
a = np.linspace(10, 100, 12)
b = np.random.randint(10,100,12)
c = np.linspace(5,20,12)
d = np.random.randint(100,150,12)
df6 = pd.DataFrame({"Var1" : a , "Var2" :b, "Var3":c, "Var4":d})
df6
df6[df6.Var2.isin([13,70])]<jupyter_output><empty_output><jupyter_text>Setting a value (column)<jupyter_code>df6.at[2:4,"Var1"]=44
df6
df6.at[2:4,"Var1"]=50
df6<jupyter_output><empty_output><jupyter_text>Setting a value (second row, second column)<jupyter_code>df6.iat[2,1]=28
df6
df6[(df6.Var1<20) & (df6.Var2>30)]<jupyter_output><empty_output><jupyter_text>Dealing with NULL values<jupyter_code>df6.at[2:3, "Var1"] = np.NaN
df6.at[5:6, "Var2"] = np.NaN
df6.at[8:10, "Var4"] = np.NaN
df6<jupyter_output><empty_output><jupyter_text>Detecting Missing Values (or df6.isna())<jupyter_code>df6.isnull()<jupyter_output><empty_output><jupyter_text>Detecting Non-Missing Values<jupyter_code>df6.notna()<jupyter_output><empty_output><jupyter_text>Sum of null:<jupyter_code>df6.isnull().sum()<jupyter_output><empty_output><jupyter_text>Filling NaNs with a specific value<jupyter_code>df6.fillna(999) #(inplace = False)<jupyter_output><empty_output><jupyter_text>Filling NaNs with different values<jupyter_code>df6.fillna(value={"Var1":333, "Var2":666,"Var4":999})<jupyter_output><empty_output><jupyter_text>Replacing based on a condition (only the first 2 NULLs in Var4 columns)<jupyter_code>df6.fillna(value={"Var4":2020}, limit=2)<jupyter_output><empty_output><jupyter_text>Dropping all NULLs<jupyter_code>df6.dropna() # default axis=0
df6.dropna(axis=1)<jupyter_output><empty_output><jupyter_text>Descriptive Statistics<jupyter_code>df6.fillna(34, inplace=True)<jupyter_output><empty_output><jupyter_text>Mean of all columns<jupyter_code>df6.mean() #(sum, min, max, median, std, var can also be used here)<jupyter_output><empty_output><jupyter_text>Mean of a single columns<jupyter_code>df6.Var3.mean()
df6.quantile([.25, .5, .75], axis = 1)
df6.quantile([.25, .5, .75], axis = 0)<jupyter_output><empty_output><jupyter_text>Describe<jupyter_code>df6.describe()
df6.describe().loc["min":"max", "Var2":"Var4"]<jupyter_output><empty_output><jupyter_text>Skew<jupyter_code>df6.skew()<jupyter_output><empty_output><jupyter_text>Kurt<jupyter_code>df6.kurt()<jupyter_output><empty_output><jupyter_text>Correlation<jupyter_code>df6.corr()<jupyter_output><empty_output><jupyter_text>Covarience<jupyter_code>df6.cov()<jupyter_output><empty_output><jupyter_text>Usage of different functions<jupyter_code>df6.apply(sum) #(min, max, mean, median and other aggs can be used here)
df6.apply(np.sum, axis=0)
df6.apply(np.sum, axis=1)
df6.applymap(np.sqrt)<jupyter_output><empty_output><jupyter_text>Lambda Functions<jupyter_code>df6.apply(lambda x: x*2)
df6.apply(lambda x: max(x))   # Default axis=0
df6.apply(lambda x: max(x), axis=1)<jupyter_output><empty_output><jupyter_text>Merging Dataframes<jupyter_code>dfstd
dfgrad =pd.DataFrame({"Student_id":[101, 103, 105, 107, 109], "Grades": [80, 85, 90, 100, 95]})
dfgrad<jupyter_output><empty_output><jupyter_text>Join Type-1: Inner Join<jupyter_code>pd.merge(dfstd, dfgrad, on="Student_id", how="inner")<jupyter_output><empty_output><jupyter_text>Join Type-2:Left Join<jupyter_code>pd.merge(dfstd, dfgrad, on="Student_id", how="left")<jupyter_output><empty_output><jupyter_text>Join Type-3:Right Join<jupyter_code>pd.merge(dfstd, dfgrad, on="Student_id", how="right")<jupyter_output><empty_output><jupyter_text>Join Type-4:Outer Join<jupyter_code>pd.merge(dfstd, dfgrad, on="Student_id", how="outer")<jupyter_output><empty_output><jupyter_text>Loading a dataset from seaborn<jupyter_code>import seaborn as sns
data=sns.load_dataset("titanic")
data.head()
data.tail()<jupyter_output><empty_output><jupyter_text>Another version instead of tail()<jupyter_code>data[-5:]
data.info()
data.embarked.unique()
data.embarked.nunique()
data[["pclass", "survived", "sex"]].head()
data.iloc[0:5, 0:3]<jupyter_output><empty_output><jupyter_text>Filtering data<jupyter_code>data.loc[data.sex=="male"]<jupyter_output><empty_output><jupyter_text>Filtering data<jupyter_code>data.loc[(data.sex=="male") & (data.pclass==3)]<jupyter_output><empty_output><jupyter_text>Displaying sorted partition of the dataframe<jupyter_code>display("Top 3 most expensive tickets:", data.sort_values(by="fare", ascending=False).head(3))<jupyter_output><empty_output><jupyter_text>Operations with str<jupyter_code>data2=sns.load_dataset("mpg")
data2.head(2)<jupyter_output><empty_output><jupyter_text>Contains<jupyter_code>data2.name.str.contains("chevrolet").count()
data2.loc[data2.name.str.contains("chevrolet")].head(3)
data2.loc[~data2.name.str.contains("chevrolet")].head()
data2.loc[(data2.name.str.contains("chevrolet")) & (data2.origin.str.contains("usa"))].head()
data2.loc[data2.name.str.contains("chevrolet|buick", regex=True)].head()
data2.loc[data2.name.str.contains("CHEVROLET|BUICK", case=False, regex=True)].head()<jupyter_output><empty_output><jupyter_text>Contains with regex all name values starting with "bu"<jupyter_code>data2.loc[data2.name.str.contains("^bu", case = False, regex = True)].head()<jupyter_output><empty_output><jupyter_text>all name values ending with "lo"<jupyter_code>data2.loc[data2.name.str.contains('lo$', case=False, regex=True)]<jupyter_output><empty_output><jupyter_text>Regex-Extracting First 5 characters from name column<jupyter_code>data2.name.str.extract(r'(^\w{5})')[160:165]<jupyter_output><empty_output><jupyter_text>Using regex-match<jupyter_code>data2.name.str.match(r'(^[C|c].*)').count()<jupyter_output><empty_output><jupyter_text>Replacing values<jupyter_code>data2.cylinders.replace({8:"eight", 6:"six"})[100:105]
data2.origin.replace(["japan", "europe"], "others")
data2.loc[data2.model_year<80, "Type"] = "Classic"
data2.head()<jupyter_output><empty_output><jupyter_text>Groupby<jupyter_code>data2.groupby("cylinders")["mpg"].mean()
data2.groupby("cylinders").mean().sort_values("mpg", ascending = False)
data2.groupby("cylinders").mpg.agg(["mean", "count"]).head()<jupyter_output><empty_output><jupyter_text>Selecting data based on dtypes<jupyter_code>data2.select_dtypes(include='number').head()
data2.select_dtypes(include = ["number", "object"]).head()
data2.select_dtypes(exclude ="number").head()<jupyter_output><empty_output><jupyter_text>Changing data type<jupyter_code>data2.astype({'horsepower':'object', 'cylinders':'float'}).dtypes
pd.to_numeric(data2.horsepower, errors='coerce')<jupyter_output><empty_output><jupyter_text>Changing all df to numeric, filling non-nuermic values with 0<jupyter_code>data2.apply(pd.to_numeric, errors='coerce').fillna(0).head()
data2.mpg.value_counts()<jupyter_output><empty_output><jupyter_text>Converting continuous data into categorical data by cut() function<jupyter_code>pd.cut(data2.mpg, bins=[13, 16, 34, 40], labels=['low', 'medium', 'high']).head(10)<jupyter_output><empty_output><jupyter_text>Unstack<jupyter_code>data2.groupby(['cylinders', 'origin']).mpg.mean().unstack()<jupyter_output><empty_output><jupyter_text>Pivot table<jupyter_code>data2.pivot_table(index='cylinders', columns='origin', values='mpg', aggfunc='mean')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup