# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Download Documents via Scrape from <a href="http://dof.ca.gov/Forecasting/Economics/Economic_and_Revenue_Updates/">this site</a>
#
# ```http://dof.ca.gov/Forecasting/Economics/Economic_and_Revenue_Updates/```

import requests
from bs4 import BeautifulSoup 

## url to scrape
url = "http://dof.ca.gov/Forecasting/Economics/Economic_and_Revenue_Updates/"

## get url and print but hard to read. will do prettify next
page = requests.get(url)
soup = BeautifulSoup(page.content, "html.parser")
print(soup)

## gather all the div with class = "quarter"
pdf_section = soup.find_all("ul", class_="archive")
print(pdf_section)

## check type
print(type(pdf_section))

# +
## gather each li in the class quarter using a for loop
all_li_fl = []
for each_section in pdf_section:
    all_li_fl.append(each_section.find_all("li"))

    
all_li_fl
# -

## we do the same but in a list comprehension
all_uls = [each_section.find_all("li") for each_section in pdf_section]
all_uls

# ## Notice how you have lists within a big list
#
# This will require you to run a for loop within a for loop

## base url we'll need to add to partial url
base_url = "http://dof.ca.gov/Forecasting/Economics/Economic_and_Revenue_Updates/"

# +
import itertools

all_urls_it = [base_url + target.find("a").get("href")\
               for target in list(itertools.chain(*all_uls))]
all_urls_it

# +
## a for loop within a for loop to target the lists within the big list
all_urls_fl = []
for my_LIs in all_uls:
    for myLi in my_LIs:
        all_urls_fl.append(base_url+myLi.find("a").get("href"))
        
        


all_urls_fl
# -

import time # time is required. we will use its sleep function
from random import randrange # generate random numbers
import wget # can put down documents, files from websites

## download all PDFs
links_number = len(all_urls_it)
link_count = 1
for link in all_urls_it:
    print(f"Downloaded link {link_count} of {links_number}")
    wget.download(link, "")
    link_count += 1
    snooze = randrange(3,6)
    print(f"snoozing for {snooze} seconds before scraping next link.")

StarcoderdataJupyterScriptsDedupFiltered