Run spider against a URL.
163132
def run_spider(self, target_url, context_name=None, user_name=None):
        """"""
        self.logger.debug('Spidering target {0}...'.format(target_url))

        context_id, user_id = self._get_context_and_user_ids(context_name, user_name)

        if user_id:
            self.logger.debug('Running spider in context {0} as user {1}'.format(context_id, user_id))
            scan_id = self.zap.spider.scan_as_user(context_id, user_id, target_url)
        else:
            scan_id = self.zap.spider.scan(target_url)

        if not scan_id:
            raise ZAPError('Error running spider.')
        elif not scan_id.isdigit():
            raise ZAPError('Error running spider: "{0}"'.format(scan_id))

        self.logger.debug('Started spider with ID {0}...'.format(scan_id))

        while int(self.zap.spider.status()) < 100:
            self.logger.debug('Spider progress %: {0}'.format(self.zap.spider.status()))
            time.sleep(self._status_check_sleep)

        self.logger.debug('Spider #{0} completed'.format(scan_id))
832351
def _run__http(self, action, replace):
        """"""

        query = action['query']
#        self._debug = True
        url = '{type}://{host}{path}'.format(path=query['path'], **action)
        content = None
        method = query.get('method', "get").lower()
        self.debug("{} {} url={}\n", action['type'], method, url)
        if method == "post":
            content = query['content']
        headers = query.get('headers', {})

        if replace and action.get('template'):
            self.rfxcfg.macro_expand(url, replace)
            if content:
                if isinstance(content, dict):
                    for key, value in content.items():
                        content[key] = self.rfxcfg.macro_expand(value, replace)
                else:
                    content = self.rfxcfg.macro_expand(content, replace)

            newhdrs = dict()
            for key, value in headers.items():
                newhdrs[key.lower()] = self.rfxcfg.macro_expand(value, replace)
            headers = newhdrs

        self.debug("{} headers={}\n", action['type'], headers)
        self.debug("{} content={}\n", action['type'], content)

        if content and isinstance(content, dict):
            content = json.dumps(content)

        self.logf("Action {name} {type}\n", **action)
        result = getattr(requests, method)(url, headers=headers, data=content, timeout=action.get('timeout', 5))
        expect = action.get('expect', {})
        expected_codes = expect.get("response-codes", (200, 201, 202, 204))
        self.debug("{} expect codes={}\n", action['type'], expected_codes)
        self.debug("{} status={} content={}\n", action['type'], result.status_code, result.text)
        if result.status_code not in expected_codes:
            self.die("Unable to make {} call, unexpected result ({})",
                     action['type'], result.status_code)

        if 'content' in expect:
            self.debug("{} expect content={}\n", action['type'], expect['content'])
            if expect['content'] not in result.text:
                self.die("{} call to {} failed\nExpected: {}\nReceived:\n{}",
                         action['type'], url, expect['content'], result.text)

        if 'regex' in expect:
            self.debug("{} expect regex={}\n", action['type'], expect['regex'])
            if not re.search(expect['regex'], result.text):
                self.die("{} call to {} failed\nRegex: {}\nDid not match:\n{}",
                         action['type'], url, expect['regex'], result.text)

        self.log(result.text, level=common.log_msg)

        self.logf("Success, status={}\n", result.status_code, level=common.log_good)
        return True
576172
def url_should_be(self, url):
    """"""

    if world.browser.current_url != url:
        raise AssertionError(
            "Browser URL expected to be {!r}, got {!r}.".format(
                url, world.browser.current_url))
186464
def extract(self, url=None, raw_html=None):
        
        cc = CrawlCandidate(self.config, url, raw_html)
        return self.crawl(cc)
320078
public void goRunWithHttpd(int port, String... crawlerNames) {
        goRun(false, crawlerNames);
        Map<String, CrawlerModel> crawlerModelContext = CrawlerCache.getCrawlerModelContext();
        SeimiHttpHandler seimiHttpHandler = new SeimiHttpHandler(crawlerModelContext);
        if (crawlerNames == null || crawlerNames.length == 0) {
            for (Map.Entry<String, CrawlerModel> entry : crawlerModelContext.entrySet()) {
                seimiHttpHandler.add("/push/" + entry.getKey(), new PushRequestHttpProcessor(entry.getValue().getQueueInstance(), entry.getKey()))
                        .add("/status/" + entry.getKey(), new CrawlerStatusHttpProcessor(entry.getValue().getQueueInstance(), entry.getKey()));
            }
        } else {
            for (String name : crawlerNames) {
                CrawlerModel crawlerModel = crawlerModelContext.get(name);
                if (crawlerModel != null) {
                    seimiHttpHandler.add("/push/" + name, new PushRequestHttpProcessor(crawlerModel.getQueueInstance(), name))
                            .add("/status/" + name, new CrawlerStatusHttpProcessor(crawlerModel.getQueueInstance(), name));
                }
            }
        }
        logger.info("Http request push service also started on port:{}", port);
        startJetty(port, seimiHttpHandler);
    }
1024607
def urlretrieve(url, dest, write_mode="w"):
    """"""
    response = urllib2.urlopen(url)
    mkdir_recursive(os.path.dirname(dest))
    with open(dest, write_mode) as f:
        f.write(response.read())
        f.close()
865771
def shouldSkipUrl(self, url, data):
        """"""
        return url in (
            self.stripUrl % '130217', # video
            self.stripUrl % '130218', # video
            self.stripUrl % '130226', # video
            self.stripUrl % '130424', # video
        )
39064
def run_web(self, flask, host='127.0.0.1', port=5000, **options):
        # type: (Zsl, str, int, **Any)->None
        """"""
        return flask.run(
            host=flask.config.get('FLASK_HOST', host),
            port=flask.config.get('FLASK_PORT', port),
            debug=flask.config.get('DEBUG', False),
            **options
        )
554212
def handle(self, key, value):
        '''
        
        '''
        # very similar to stop
        # break down key
        elements = key.split(":")
        spiderid = elements[1]
        appid = elements[2]
        crawlid = elements[3]

        # log ack of expire
        extras = self.get_log_dict('expire', appid,
                                   spiderid, crawlid=crawlid)
        self.logger.info("Expiring crawl found", extra=extras)

        # add crawl to blacklist so it doesnt propagate
        redis_key = spiderid + ":blacklist"
        value = '{appid}||{crawlid}'.format(appid=appid,
                                            crawlid=crawlid)
        # add this to the blacklist set
        self.redis_conn.sadd(redis_key, value)

        # everything stored in the queue is now expired
        result = self._purge_crawl(spiderid, appid, crawlid)

        # add result to our dict
        master = {}
        master['server_time'] = int(self.get_current_time())
        master['crawlid'] = crawlid
        master['spiderid'] = spiderid
        master['appid'] = appid
        master['total_expired'] = result
        master['action'] = 'expired'

        if self._send_to_kafka(master):
            master['success'] = True
            self.logger.info('Sent expired ack to kafka', extra=master)
        else:
            master['success'] = False
            self.logger.error('Failed to send expired ack to kafka',
                              extra=master)
626169
def crawl(self):
        """"""
        page = Linkfetcher(self.root)
        page.linkfetch()
        queue = Queue()
        for url in page.urls:
            queue.put(url)
        followed = [self.root]

        n = 0

        while True:
            try:
                url = queue.get()
            except QueueEmpty:
                break

            n += 1

            if url not in followed:
                try:

                    host = urllib.parse.urlparse(url)[1]

                    if self.locked and re.match(".*%s" % self.host, host):
                        followed.append(url)
                        self.followed += 1
                        page = Linkfetcher(url)
                        page.linkfetch()
                        for i, url in enumerate(page):
                            if url not in self.urls:
                                self.links += 1
                                queue.put(url)
                                self.urls.append(url)

                        if n > self.depth and self.depth > 0:
                            break
                except Exception as e:
                    print("ERROR: The URL '%s' can't be crawled (%s)" % (url, e))
                    print(format_exc())
408932
def process_item(self, item, spider):  # pylint: disable=unused-argument
        ""
        url = URLObject(item["url"])
        if url.path.startswith("/api/"):
            raise DropItem(u"Dropping DRF url {url}".format(url=url))
        else:
            return item
504615
def process_response(self, request, response, spider):
        """"""
        return response.replace(body=str(BeautifulSoup(response.body, self.parser)))
250878
def handle_url(url, session, res):
    """"""
    print("Parsing", url, file=sys.stderr)
    try:
        data = getPageContent(url, session)
    except IOError as msg:
        print("ERROR:", msg, file=sys.stderr)
        return
    for match in url_matcher.finditer(data):
        comicurl = match.group(1)
        name = format_name(comicurl.split('.', 1)[0][7:])
        if name in exclude_comics:
            continue
        if contains_case_insensitive(res, name):
            # we cannot handle two comics that only differ in case
            print("INFO: skipping possible duplicate", repr(name), file=sys.stderr)
            continue
        # find out how many images this comic has
        end = match.end()
        mo = num_matcher.search(data[end:])
        if not mo:
            print("ERROR matching number:", repr(data[end:end+300]), file=sys.stderr)
            continue
        num = int(mo.group(1))
        # find activity
        mo = activity_matcher.search(data[end:])
        if not mo:
            print("ERROR matching activity:", repr(data[end:end+300]), file=sys.stderr)
            continue
        active = mo.group(1).lower() == "active"
        res[name] = [comicurl, num, active]
    if not res:
        print("ERROR:", "did not match any comics", file=sys.stderr)
649893
def check_url(url):
        '''
        

        '''
        request = urllib2.Request(url)
        try:
            response = urlopen(request)
            return True, response.code
        except urllib2.HTTPError as e:
            return False, e.code
1106624
def checkURL(cls, trust_root, url):
        """"""
        tr = cls.parse(trust_root)
        return tr is not None and tr.validateURL(url)
483849
def visit(self, url=''):
        
        def _visit(url):
            if len(url) > 0 and url[0] == '/':
                # url's first character is a forward slash; treat as relative path
                path = url
                full_url = self.driver.current_url
                parsed_url = urlparse(full_url)
                base_url = str(parsed_url.scheme) + '://' + str(parsed_url.netloc)
                url = urljoin(base_url, path)

            try:
                return self.driver.get(url)
            except TimeoutException:
                if self.ignore_page_load_timeouts:
                    pass
                else:
                    raise PageTimeoutException.PageTimeoutException(self, url)

        return self.execute_and_handle_webdriver_exceptions(lambda: _visit(url))
1094203
def get_action(args):
    """"""
    redirect_mode = args.bang or args.search or args.lucky
    if redirect_mode and not args.url:
        return webbrowser.open_new_tab
    else:
        return print_result