# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/harsh3375/Bank-Marketing/blob/master/upchar_extract.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="Y7JhFHZiisse"


# + [markdown] id="j7HxRKLBcTel"
#
# using BeautifulSoup to Extract Data from Webpage
#
#
#

# + id="wjlG4uoYZVtU" colab={"base_uri": "https://localhost:8080/"} outputId="50677183-8312-4682-9803-181d38e07ea3"
# !pip install requests --upgrade --quiet

# + id="EmjrvYO20oAN"
import requests

# + id="Eu6h4Vrr0r3f"
topics_url = 'https://www.myupchar.com/en/search?term=Paracetamol%20&filter=medicine'

# + id="cTJX24WR3WGJ"
response= requests.get(topics_url)

# + colab={"base_uri": "https://localhost:8080/"} id="u-96p1re3lMD" outputId="84c4aa86-69ef-4e50-ae72-18901b2c317d"
response.status_code

# + colab={"base_uri": "https://localhost:8080/"} id="CV-4vc5n3rGF" outputId="fcd8c366-7dac-4537-8acc-eaa25cd69323"
len(response.text)

# + id="V5Eqkm9J4BpA"
page_contents= response.text

# + colab={"base_uri": "https://localhost:8080/", "height": 137} id="vzkwkfwJ4USn" outputId="3d2f2b85-0bdf-4536-9ec6-1bc9f9353238"
page_contents[:1000]

# + id="oox5rZH-4XTN"
with open('webpage.html', 'w') as f:
  f.write(page_contents)

# + colab={"base_uri": "https://localhost:8080/"} id="D7AEh5jW5Vwy" outputId="417e51a1-97c9-44d6-c28b-3b2cd70528d7"
# !pip install beautifulsoup4 --upgrade --quiet



# + id="EFjbmpkb4itL"
from bs4 import BeautifulSoup

# + id="4xFUWcwp4ivM"
doc= BeautifulSoup(page_contents,'html.parser')

# + id="lP0_qFXt4i1A"
title_tag= doc.find_all('h5')

# + colab={"base_uri": "https://localhost:8080/"} id="m9ESkklj4i33" outputId="0effb3ac-a63d-4a1e-8931-70800cf49123"
title_tag

# + colab={"base_uri": "https://localhost:8080/"} id="NxbaSgao4i6t" outputId="fd4d229d-5dc3-465d-8034-582a406279e5"
title_tag
titles= []
for tag in title_tag:
  titles.append(tag.text)
print(titles[:])

# + id="0K2MtXwh4i9c" colab={"base_uri": "https://localhost:8080/"} outputId="997e3dfb-3cc0-409c-dce4-4cd6864f4192"


# + colab={"base_uri": "https://localhost:8080/"} id="dFoqtMqj4i_L" outputId="f509780e-20bd-443e-fe06-71f3699dea25"
len(titles)

# + [markdown] id="lVvVYxnW_HK5"
# topic urls

# + id="T26tRE1F-I5e"
link_tags=  titles

# + id="FyyGtCdL_FZm" colab={"base_uri": "https://localhost:8080/", "height": 163} outputId="8e40944f-8569-42e8-ee23-7098538fd9d4"
div_tags=title_tags0.parent.parent

# + id="kiCqWtV-_Ff6"
link_tags= doc.find_all('a')

# + id="7-K-m2aqoaRS"
title_tag1=doc.find_all('a', {'class': 'surgery-section-bx'})

# + id="uSOR9a-H_FiE"
selection_class = 'surgery-section-bx'

title_tags= doc.find_all('a', {'class': selection_class})

# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="biD9sXwQ_Fk2" outputId="ae56f5a7-df57-4e77-a3fd-44e1bec159de"
title_tags[0]['href']

# + id="VSMUWo7M_FnA"
topic_urls0='https://www.myupchar.com/' + title_tags[0]['href']

# + id="aZYVrBkvyT1r"


# + colab={"base_uri": "https://localhost:8080/"} id="slZR478X_Fq-" outputId="51be9426-cd4b-48d6-b043-6198506cd8d7"
len(topic_urls0)

# + colab={"base_uri": "https://localhost:8080/"} id="bJDkL1s6CH_S" outputId="1ae7eed7-2756-455e-b052-9f51ab47ee08"
print(topic_urls0)

# + id="8mOAi7B1CM8p"
upchar_urls = []
base_url = 'https://www.myupchar.com/'
for tag in title_tags:
  upchar_urls.append(base_url + tag['href'])
  

# + colab={"base_uri": "https://localhost:8080/"} id="HojjVj1FCaeg" outputId="46801ee4-919f-4d96-d519-a50b766d2631"
upchar_urls

# + colab={"base_uri": "https://localhost:8080/"} id="7nzc88z6idrV" outputId="ed1819d4-cb46-4420-ede8-e7bdb4f0c2ef"
len(titles)

# + id="cORBLxB82GC2"
item_dict= {'Brand Name': titles ,
             'URL': upchar_urls}

# + colab={"base_uri": "https://localhost:8080/"} id="jlPNA3jL2eBT" outputId="312e77c5-79b9-4bcf-d2d0-fd8cbd04cb2f"
titles

# + id="_P9zdkuk15Qs"
ext= pd.DataFrame(item_dict)

# + colab={"base_uri": "https://localhost:8080/", "height": 793} id="bSWyOQwG4tHm" outputId="4623b1a3-72b1-4f01-a1a2-34e798898dd1"
ext


# + id="5nS6QrX3zHdq"
from bs4 import BeautifulSoup
doc= BeautifulSoup(page_contents,'html.parser')

# + id="1_MpxMtQzHjp"


# + id="HcRFqCk2zHms"


# + id="-F5RblvczHo6"


# + id="2e-ZF-FUzHrT"


# + id="hXDbiJ2szHuR"


# + id="a7JFl9iSzHwy"


# + id="9nfgJn5dzH00"


# + id="bpwnaxiRzH2u"


# + id="1r_c4r9pidrX"


# + id="Vrmzse3GckRt"


# + id="g1v9nzLNckUT"


# + id="_JXtLwMxckXg"


StarcoderdataJupyterScriptsDedupFiltered