<reponame>1695652161/Spider_Armies<filename>quotes.toscrape.com/tutorial/tutorial/spiders/quotes.py
from typing import Counter
import scrapy
from tutorial.items import QuoteItem
from scrapy.http.request import Request


class QuotesSpider(scrapy.Spider):
    # 爬虫名称, 唯一的

    name = 'quotes'

    # 请求url非该域名则过滤
    # allowed_domains = ['quotes.toscrape.com']

    # is_open_count = True
    # count = 0
    # MAX = 5

    custom_settings = {
<<<<<<< HEAD
        "CONCURRENT_REQUESTS": 6,
        "DOWNLOAD_DELAY": 0,
        'tutorial.middlewares.TutorialDownloaderMiddleware': 543,
        'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
        'tutorial.middlewares.TutorialRetryMiddleware': 550,

    }

    start_urls = [
        'http://httpbin.org/ip',
        'http://httpbin.org/ip',
        'http://httpbin.org/ip',
        'http://httpbin.org/ip',
        'http://httpbin.org/ip',
        'http://httpbin.org/ip',
    ]

    # start_urls = [
    #     # 'http://quotes.toscrape.com/page/1/',
    #     # 'http://quotes.toscrape.com/page/2/',
    #     # 'http://quotes.toscrape.com/page/3/',
    #     # 'http://quotes.toscrape.com/page/4/',
    #     # 'http://quotes.toscrape.com/page/5/',
    #     # 'http://quotes.toscrape.com/page/6/',
    #     # 'http://quotes.toscrape.com/page/7/',
    #     # 'http://quotes.toscrape.com/page/8/',
    #     # 'http://quotes.toscrape.com/page/9/',
    #     # 'http://quotes.toscrape.com/page/10/',
    #     # 'http://quotes.toscrape.com/page/11/',
    #     # 'http://quotes.toscrape.com/page/12/',
    #     # 'http://quotes.toscrape.com/page/13/',
    #     # 'http://quotes.toscrape.com/page/14/',
    #     # 'http://quotes.toscrape.com/page/15/',
    #     # 'http://quotes.toscrape.com/page/16/',
    #     # 'http://quotes.toscrape.com/page/17/',
    #     # 'https://www.correos.cl/',
    #     # 'https://www.correos.cl/',
    #     # 'https://www.correos.cl/',
=======
        "CONCURRENT_REQUESTS": 4,
        "DOWNLOAD_DELAY":0.5,
    }

    start_urls = [
        'http://httpbin.org/ip#1/',
        'http://httpbin.org/ip#2/',
        'http://httpbin.org/ip#3/',
        'http://httpbin.org/ip#4/',
        'http://httpbin.org/ip#5/',
        'http://httpbin.org/ip#6/',
        'http://httpbin.org/ip#7/',
        'http://httpbin.org/ip#8/',
        'http://httpbin.org/ip#9/',
        'http://httpbin.org/ip#10/',
        'http://httpbin.org/ip#11/',
        'http://httpbin.org/ip#12/',
        'http://httpbin.org/ip#13/',
        'http://httpbin.org/ip#14/',
        'http://httpbin.org/ip#15/',
        'http://httpbin.org/ip#16/',
        'http://httpbin.org/ip#17/',
        'http://httpbin.org/ip#17/',
        'http://httpbin.org/ip#18/',
        'http://httpbin.org/ip#19/',
        'http://httpbin.org/ip#20/',
        'http://httpbin.org/ip#21/',
    ]

    # start_urls = [
    #     'http://quotes.toscrape.com/page/1/',
    #     'http://quotes.toscrape.com/page/2/',
    #     'http://quotes.toscrape.com/page/3/',
    #     'http://quotes.toscrape.com/page/4/',
    #     'http://quotes.toscrape.com/page/5/',
    #     'http://quotes.toscrape.com/page/6/',
    #     'http://quotes.toscrape.com/page/7/',
    #     'http://quotes.toscrape.com/page/8/',
    #     'http://quotes.toscrape.com/page/9/',
    #     'http://quotes.toscrape.com/page/10/',
    #     'http://quotes.toscrape.com/page/11/',
    #     'http://quotes.toscrape.com/page/12/',
    #     'http://quotes.toscrape.com/page/13/',
    #     'http://quotes.toscrape.com/page/14/',
    #     'http://quotes.toscrape.com/page/15/',
    #     'http://quotes.toscrape.com/page/16/',
    #     'http://quotes.toscrape.com/page/17/',
    #     'https://www.correos.cl/',
    #     'https://www.correos.cl/',
    #     'https://www.correos.cl/',
>>>>>>> 0b240f3f443ce7cf1346e781b65bff5ca72101fb
    # ]


    def parse(self, response):

        item = QuoteItem()
        item['url'] = response.url
        item['data'] = response.body.decode()
        # print(response.body.decode())
        return item

