# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Web scraping using pandas
#
# pandas provides a `read_html` function which is convenient to parse tables from web pages. As an example, we will use the https://en.wikipedia.org/wiki/List_of_ICAO_aircraft_type_designators#cite_note-ICAOcode-2 to get IATA and ICAO codes for aircrats.

import pandas as pd

out = pd.read_html("https://en.wikipedia.org/wiki/List_of_ICAO_aircraft_type_designators#cite_note-ICAOcode-2")

# This returns a list of all tables found in the page:

print(type(out))

# Each item in this list is a dataframe:

out[0].head()

# We can see that first row holds labels of columns, we can fix this easilly:

df = out[0].drop(axis=0, labels=0)
df.columns = ["ICAO", "IATA", "Fullname"]

df.head()

# We can clean the dataset by removing duplicates (if any):

df = df[~df.duplicated()]


# Split manufacturer from model:

def split(row):
    series = row.copy()
    fields = series.Fullname.split()
    try:
        field2 = fields[2]
    except IndexError:
        field2 = None
    try:
        field3 = fields[3]
    except IndexError:
        field3 = None
    if series.Fullname == "Launch - Boat":
        series["Manufacturer"] = u"Launch"
        series["Model"] = u"Boat"
    elif series.Fullname == "Road Feeder Service - Cargo Truck":
        series["Manufacturer"] = u"Road Feeder Service"
        series["Model"] = u"Cargo Truck"
    elif series.Fullname == "Concorde":
        series["Manufacturer"] = u"Aerospatiale"
        series["Model"] = u"Concorde"
    elif series.Fullname == "EV-97 EuroStar / EV-97 Harmony":
        series["Manufacturer"] = u"Evektor-Aerotechnik"
        series["Model"] = u"EV-97 EuroStar / EV-97 Harmony"
    elif series.Fullname == "Evektor SportStar":
        series["Manufacturer"] = u"Evektor-Aerotechnik"
        series["Model"] = u"SportStar"
    elif series.Fullname == "Gulfstream Aerospace G-159 Gulfstream I":
        series["Manufacturer"] = u"Grumman"
        series["Model"] = u"G-159 Gulfstream I"
    elif fields[0] == "Pipistrel":
        series["Manufacturer"] = u"Pipistrel"
        series["Model"] = fields[1].rstrip('01234569789')
    elif len(fields) == 1:
        series["Manufacturer"] = series.Fullname
        series["Model"] = series.Fullname
    elif field3 in ["Siddeley)"]:
        series["Manufacturer"] = " ".join(fields[0:4])
        series["Model"] = " ".join(fields[4:])
    elif field2 in ["Factories", "Aviation)", "IPTN", "Mystere)", "Canada", "Industries", "(BAC)"]:
        series["Manufacturer"] = u" ".join(fields[0:3])
        series["Model"] = u" ".join(fields[3:])
    elif fields[1] in ["Industrie", "(Aero)", "(Aerospatiale)", "Aerospace", "(BAC)", "Britten-Norman", 
                       "Dornier", "Havilland", "Siddeley", "Douglas", "(MBB)", "Helicopters", "(Nord)", 
                       "(Swearingen)", "Yunshuji", "Ultralight", "Rocket"]:
        series["Manufacturer"] = u" ".join(fields[0:2])
        series["Model"] = u" ".join(fields[2:])
    elif fields[0] in ["Fokker", "BAe", "Airbus", "Boeing", "Antonov", "Agusta", "Ayres", "Avro", 
                       "Aerospatiale/Alenia", "Beechcraft", "Bell", "Canadair", "Cessna", "Convair", 
                       "Curtiss", "Douglas", "Embraer", "Eurocopter", "Fairchild", "Grumman", "Helio",
                       "Ilyushin", "Junkers", "Lockheed", "LET", "Gates", "MIL", "Mitsubishi", "Piper",
                       "Pilatus", "Partenavia", "Saab", "Sikorsky", "Shorts", "Aerospatiale/BAC", 
                       "Sukhoi", "Tupolev", "Vickers", "Yakovlev", "NAMC", "Beechcfrat", "Aerospatiale", 
                       "ICON", "Aeroprakt", "Robin", "Bombardier", "Beriev", "COMAC", "CASA/IPTN", 
                       "Tecnam", "Dassault", "Gulfstream", "Honda", "Learjet", "Mil", "Pipistrel", 
                       "Reims-Cessna", "Socata"]:
        series["Manufacturer"] = fields[0]
        series["Model"] = u" ".join(fields[1:])
    else:
        series["Manufacturer"] = series.Fullname
        series["Model"] = series.Fullname
    series.drop(labels="Fullname", inplace=True)
    return series


df2 = df.apply(split, axis=1)

# And now let's map manufacturer names:

df2.Manufacturer.replace({u"BAe": u"British Aerospace",
                          u"British Aerospace (BAC)": u"British Aerospace",
                          u"Airbus Industrie": u"Airbus", 
                          u"Gulfstream/Rockwell (Aero)": u"Gulfstream/Rockwell",
                          u"Gulfstream/Rockwell": u"Gulfstream/Rockwell",
                          u"Eurocopter (Aerospatiale)": u"Eurocopter",
                          u"Eurocopter (MBB)": u"Eurocopter", 
                          u"Beechcfrat": u"Beechcraft", 
                          u"Aerospatiale (Sud Aviation)": u"Aerospatiale",
                          u"Aerospatiale (Nord)": u"Aerospatiale",
                          u"Aerospatiale/Alenia": u"ATR",
                          u"CASA / IPTN": u"CASA/IPTN",
                          u"Dassault (Breguet Mystere)": u"Dassault",
                          u"MIL": u"Mil",
                          u"De Havilland Canada": u"De Havilland"}, inplace=True)

manuf = df2.Manufacturer.unique()
manuf.sort()
print(manuf)

models = df2.Model.unique()
models.sort()
print(models)

# Now save the dataset for a later re-use:

df2.to_csv("../data/aircrafts_codes.csv", encoding='utf-8', index=False)



StarcoderdataJupyterScriptsDedupFiltered