# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Scraping the Data

# ### I used the BeautifulSoup package to transform the data in the table on the Wikipedia page into pandas dataframe

import requests
website_url = requests.get('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M').text

from bs4 import BeautifulSoup
soup = BeautifulSoup(website_url,'lxml')

# ### We can observe that the tabular data is availabe in table and belongs to class="wikitable sortable", So we need to extract only table

Minha_tabela = soup.find('table',{'class':'wikitable sortable'})

print(Minha_tabela.tr.text)

headers="Postcode,Borough,Neighbourhood"

# ### Geting the values in tr and separate each td within by ","
#

tabela1=""
for tr in Minha_tabela.find_all('tr'):
    row1=""
    for tds in tr.find_all('td'):
        row1=row1+","+tds.text
    tabela1=tabela1+row1[1:]
print(tabela1)

# ### Store the data in .csv file
#

file=open("toronto-data.csv","wb")
file.write(bytes(tabela1,encoding="ascii",errors="ignore"))

# ### Converting into dataframe and assigning column names

import pandas as pd
df = pd.read_csv('toronto-data.csv',header=None)
df.columns=["Postalcode","Borough","Neighbourhood"]

df.head(10)

# ### Drop row where borough is "Not assigned"

# +
# Get names of indexes for which column Borough has value "Not assigned"
indexNames = df[ df['Borough'] =='Not assigned'].index

# Delete these row indexes from dataFrame
df.drop(indexNames , inplace=True)
# -

df.head(10)

# ### If a cell has a borough but a Not assigned neighborhood, then the neighborhood will be the same as the borough

df.loc[df['Neighbourhood'] =='Not assigned' , 'Neighbourhood'] = df['Borough']
df.head(10)

#
# ### Rows will be same postalcode will combined into one row with the neighborhoods separated with a comma
#

result = df.groupby(['Postalcode','Borough'], sort=False).agg( ', '.join)

df_new=result.reset_index()
df_new.head(10)

# ## Store data in new .csv with clean data.

df_new.to_csv(r'Toronto-data-processed.csv', index = False, header = True)

# #### That's It, Thank you :)



StarcoderdataJupyterScriptsDedupFiltered