# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import requests
from bs4 import BeautifulSoup as bs4
import re

# +
# create beautifulsoup object
html = """
<html>
    <h1> Test </h1>
    <p> Hello 1 </p>
    <p> World 2 </p>
</html>"""

soup = bs4(html, 'html.parser') # BS object
soup('p') # parse HTML anchors 'p'
# -

url = 'http://research.atmos.ucla.edu/csrl/pub.html'
page = requests.get(url) # use requests to send a GET request to the assigned url

# - create a variable for the target url
# - use request to get the url

page.text # returns the html text in string format

soup = bs4(page.text, 'html.parser') #beautifulsoup object

soup.html # beautifulsoup object to parse html; type is bs4.element.Tag

links = [] # create an empty container to store list of links
for url in soup.find_all('a', text=re.compile('download')):
    links.append(url['href'])
    len(links)

# * create an empty list [] variable `links` so we can store the links
# * loop through the `soup.find_all` command that pulls out the html `a` links
#     - Note: `soup.find_all` returns a `list` object that we can use in a loop
# * for each `a` tag append the link with `link['href']`
# * this in total is called the [accumulator pattern](https://interactivepython.org/runestone/static/pip2/Iteration/TheAccumulatorPattern.html): we loop through some collection and pull out parts of it and add to a list or another data type

links

for url in links:
    print(url)

# lets test to see if we can isolate our dois
for url in links:
    if 'doi' in url:
        print(url)
    else:
        pass

# lets see if we can pull out dois
dois = []
pattern = re.compile('\/(10\.\d{4}\/.*$)')
for url in links: #in url in each link
    if 'doi' in url: # if
            match = pattern.search(url)
            print(match.group(1))
            dois.append(match.group(1))
    else:
        pass

# * for URL in each link
# * if URL contains 'doi', search the list for the RegEx pattern above; assign as `match.group(1)` and print
# * to `dois` list, append each match result

base_url = "https://api.crossref.org/works/"
pubs = []
for doi in dois:
    article = {}
    #construct query to crossref
    #print(doi)
    work = requests.get(base_url + doi)
    if work.ok:
        #building a dictionary for each pub that has parts of what we want
        article['title'] = work.json()['message']['title'][0]
        #article['authors'] = work.json()['message']['author']
        article['URL'] = work.json()['message']['URL']
        pubs.append(article)

pubs

import json
with open("pubs2.json", 'w') as filehandle:
    json.dump(pubs, filehandle)

base_url = "https://api.crossref.org/works/"
#print(base_url + dois[0])
work = requests.get(base_url + '10.1007/s00382-015-2846-0')

pub = {}
pub['title'] = work.json()['message']['title'][0]
pub['authors'] = work.json()['message']['author']
pub['URL'] = work.json()['message']['URL']


pub



StarcoderdataJupyterScriptsDedupFiltered