<jupyter_start><jupyter_text>Mission to Mars - web scraping<jupyter_code>from splinter import Browser
from bs4 import BeautifulSoup
import pandas as pd
import requests
import time
# Connect to the Chrome browser

executable_path = {"executable_path": "C:/Users/cindy/Downloads/chromedriver_win32/chromedriver.exe"}
browser = Browser("chrome", **executable_path, headless=False)<jupyter_output><empty_output><jupyter_text>NASA Mars News<jupyter_code>url = "https://mars.nasa.gov/news/"
browser.visit(url)
html = browser.html

soup = BeautifulSoup(html, "html.parser")
# print(soup.prettify())

# find most recent news title from mars.nasa.gov

news_title = soup.find_all('div', class_='content_title')[1].text
print(news_title)
# find the paragraph for the most recent news title from mars.nasa.gov

news_p = soup.find('div', class_='article_teaser_body').text    
print(news_p)<jupyter_output>Like much of the rest of the world, the Mars rover team is pushing forward with its mission-critical work while putting the health and safety of their colleagues and community first.<jupyter_text>JPL Mars Space Images - Featured Image<jupyter_code>url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
browser.visit(url)
html = browser.html

# click the button to get to the page with the full image, wait 5 seconds for the page to load and click next button

browser.click_link_by_partial_text('FULL IMAGE')
time.sleep(5)

browser.click_link_by_partial_text('more info')
html = browser.html
soup_jpl = BeautifulSoup(html, "html.parser")
# print(soup_jpl.prettify())
featured_url_image1 = soup_jpl.find('figure', class_='lede')
# print(featured_url_image1)
featured_url_image2 = featured_url_image1.find('a')['href']

featured_image_url = f'https://www.jpl.nasa.gov{featured_url_image2}'
print(featured_image_url)<jupyter_output>https://www.jpl.nasa.gov/spaceimages/images/largesize/PIA20465_hires.jpg<jupyter_text>Mars Weather twitter account<jupyter_code>#  Added in a sleep timer to allow all the data to load

url = "https://twitter.com/marswxreport?lang=en"
browser.visit(url)

time.sleep(10)

html = browser.html
soup_twitter = BeautifulSoup(html, "html.parser")
# print(soup_twitter.prettify())
# re is Python 'regular expression' programming language embedded in Python to perform matching.
import re
pattern = re.compile(r'sol')

mars_weather = soup_twitter.find('span', text=pattern).text
print(mars_weather)<jupyter_output>InSight sol 501 (2020-04-24) low -93.5ºC (-136.2ºF) high -4.3ºC (24.3ºF)
winds from the SW at 5.0 m/s (11.3 mph) gusting to 15.8 m/s (35.3 mph)
pressure at 6.70 hPa<jupyter_text>Mars Facts<jupyter_code># This will read HTML tables into a list of dataframe objects

url = "https://space-facts.com/mars/"
mars_list = pd.read_html(url)
mars_list2 = mars_list[2]
mars_list2
# Rename the columns
mars_list2.columns = ['Feature', 'Value']
print(mars_list2)
# put the data back into html format with html tags
mars_list2 = mars_list2.set_index('Feature')
mars_facts_html = mars_list2.to_html(classes='table table-bordered')
mars_facts_html<jupyter_output><empty_output><jupyter_text>Mars Hemispheres<jupyter_code>#  This will scrape Mars 4 hemispheres and put their name/url image link into a dictionary

url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
browser.visit(url)

time.sleep(10)

html = browser.html
soup_hemi = BeautifulSoup(html, "html.parser")
#print(soup_hemi.prettify())
hemi_list = soup_hemi.find('div', class_='collapsible results')
#print(hemi_list.prettify)
hemi_list2 = hemi_list.find_all('div', class_='item')
# print(hemi_list2)
# loop through the HTML to find the 4 hemispheres and the URL for the image.  Put into a dictionary.

hemi_dict = []

for x in hemi_list2:
    title = x.find("h3").text
    print (title)
    title = title.replace("Enhanced", "")
    link = x.find("a")["href"]
    img_link = "https://astrogeology.usgs.gov" + link
    print (img_link)
    browser.visit(img_link)
    time.sleep(5)
    html = browser.html
    soup = BeautifulSoup(html, "html.parser")
    image = soup.find('div', class_='downloads')
    image2 = image.find("a")["href"]
    hemi_dict.append({"title": title, "img url": image2})

hemi_dict<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup