# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# #! pip install scrapy
# -

from scrapy import Selector

html ='''
<html>
    <body>
        <div class="hello datacamp">
        <p>Hello World!</p>
        </div>
        <p>Enjoy DataCamp!</p>
    </body>
</html>
'''


sel= Selector(text=html)

sel.xpath("//p")

# Selector and SelectorList objects allow for chaining when using the xpath method . below are the same  21 & 22

sel.xpath('/html/body/div[1]')

sel.xpath('/html').xpath('./body/div[1]')

sel.xpath("//*")

sel.xpath('//*[contains(@class,"hello datacamp")]/p')

listext=sel.xpath("//p")[1]

listext.extract()

sel.xpath("//p").extract_first()

div=Selector(text=html)

div.xpath("//div").extract()

import requests

url= "https://www.datacamp.com/courses/all"

html=requests.get(url).content

sel=Selector(text=html)

sel.xpath('//p[contains(@class,"course-block__description")]').extract()

# +
# Import a scrapy Selector
from scrapy import Selector

# Import requests
import requests

# Create the string html containing the HTML source
html = requests.get( url ).content

# Create the Selector object sel from html
sel = Selector( text=html )

# Print out the number of elements in the HTML document
print( 'There are {0} elements in the HTML document'.format( len( sel.xpath('//*')))
print( "You have found: ", len( sel.xpath('//*') ) )
# -

sel.css('p.course-block__description').extract()

sel.xpath('//p[contains(@class,"course-block__description")]/text()').extract()

sel.xpath('//p[contains(@class,"course-block__description")]//text()').extract()

len (sel.xpath('//p[contains(@class,"course-block__description")]//text()').extract())

sel.css('p.course-block__description::text').extract()

element= sel.css('p.course-block__description::text').extract()
for elm in element:
    print(elm)

# +
# Create an XPath string to the desired text.
xpath = '//p[@id="p3"]//text()'

# Create a CSS Locator string to the desired text.
css_locator = 'p#p3 ::text'

# Print the text from our selections
print_results( xpath, css_locator )
# -

from scrapy import  Selector

# +
# Get the URL to the website loaded in response
this_url = response.url

# Get the title of the website loaded in response
this_title = response.css('title::text').extract_first()

# Print out our findings
print_url_title( this_url, this_title )

# +
# Create a SelectorList of the course titles
crs_title_els = response.css('h4::text')

# Extract the course titles 
crs_titles = crs_title_els.extract()

# Print out the course titles 
for el in crs_titles:
  print( ">>", el )
# %capture
# -

import scrapy
from scrapy.crawler import CrawlerProcess


# Create the spider class
class YourSpider(scrapy.Spider):
  name = "your_spider"
  # start_requests method
  def start_requests(self):
    pass
  # parse method
  def parse(self, response):
    pass


class DCspider( scrapy.Spider ):
    name ='dc_spider'
    def start_requests( self ):
        urls = [ 'https://www.datacamp.com/courses/all' ]
        for url in urls:
            yield scrapy.Request( url = url, callback = self.parse )
    def parse( self, response ):
        # simple example: write out the html
        html_file ='DC_courses.html'
        with open( html_file,'wb' ) as fout:
            fout.write( response.body )



# +
# Import the scrapy library
import scrapy

# Create the Spider class
class DCdescr( scrapy.Spider ):
  name = 'dcdescr'
  # start_requests method
  def start_requests( self ):
    yield scrapy.Request( url = url_short, callback = self.parse )
  
  # First parse method
  def parse( self, response ):
    links = response.css( 'div.course-block > a::attr(href)' ).extract()
    # Follow each of the extracted links
    for link in links:
      yield response.follow(url=link, callback=self.parse_descr)
      
  # Second parsing method
  def parse_descr( self,response ):
    # Extract course description
    course_descr = response.css( 'p.course__description::text' ).extract_first()
    # For now, just yield the course description
    yield course_descr


# Inspect the spider
#inspect_spider( DCdescr )

# +
# Import scrapy
import scrapy

# Import the CrawlerProcess: for running the spider
from scrapy.crawler import CrawlerProcess

# Create the Spider class
class DC_Chapter_Spider(scrapy.Spider):
  name = "dc_chapter_spider"
  # start_requests method
  def start_requests(self):
    url_shor ='https://www.datacamp.com/courses/all'
    yield scrapy.Request(url = url_short,
                         callback = self.parse_front)
  # First parsing method
  def parse_front(self, response):
    course_blocks = response.css('div.course-block')
    course_links = course_blocks.xpath('./a/@href')
    links_to_follow = course_links.extract()
    for url in links_to_follow:
      yield response.follow(url = url,
                            callback = self.parse_pages)
  # Second parsing method
  def parse_pages(self, response):
    crs_title = response.xpath('//h1[contains(@class,"title")]/text()')
    crs_title_ext = crs_title.extract_first().strip()
    ch_titles = response.css('h4.chapter__title::text')
    ch_titles_ext = [t.strip() for t in ch_titles.extract()]
    dc_dict[ crs_title_ext ] = ch_titles_ext

# Initialize the dictionary **outside** of the Spider class
dc_dict = dict()

# Run the Spider
process = CrawlerProcess()
process.crawl(DC_Chapter_Spider)
process.start()

# Print a preview of courses
#previewCourses(dc_dict)
# -

# parse method
def parse(self, response):
  # Extracted course titles
  crs_titles = response.xpath('//h4[contains(@class,"block__title")]/text()').extract()
  # Extracted course descriptions
  crs_descrs = response.xpath('//p[contains(@class,"block__description")]/text()').extract()
  # Fill in the dictionary
  for crs_title, crs_descr in zip(crs_titles, crs_descrs):
    dc_dict[crs_title] = crs_descr


# +
# Import scrapy
import scrapy

# Import the CrawlerProcess
from scrapy.crawler import CrawlerProcess

# Create the Spider class
class YourSpider(scrapy.Spider):
  name = 'yourspider'
  # start_requests method
  def start_requests( self ):
    yield scrapy.Request(url = url_short, callback=self.parse)
      
  def parse(self, response):
    # My version of the parser you wrote in the previous part
    crs_titles = response.xpath('//h4[contains(@class,"block__title")]/text()').extract()
    crs_descrs = response.xpath('//p[contains(@class,"block__description")]/text()').extract()
    for crs_title, crs_descr in zip( crs_titles, crs_descrs ):
      dc_dict[crs_title] = crs_descr
    
# Initialize the dictionary **outside** of the Spider class
dc_dict = dict()

# Run the Spider
process = CrawlerProcess()
process.crawl(YourSpider)
process.start()

# Print a preview of courses
previewCourses(dc_dict)
# -



StarcoderdataJupyterScriptsDedupFiltered