Total operations performed for this crawler
351101
def op_count(cls, crawler, stage=None):
        """"""
        if stage:
            total_ops = conn.get(make_key(crawler, stage))
        else:
            total_ops = conn.get(make_key(crawler, "total_ops"))
        return unpack_int(total_ops)
908363
def schedule_crawl(spider, workflow, **kwargs):
    """"""
    from inspire_crawler.utils import get_crawler_instance

    crawler = get_crawler_instance()
    crawler_settings = current_app.config.get('CRAWLER_SETTINGS')
    crawler_settings.update(kwargs.get("crawler_settings", {}))

    crawler_arguments = kwargs
    crawler_arguments.update(
        current_app.config.get('CRAWLER_SPIDER_ARGUMENTS', {}).get(spider, {})
    )
    job_id = crawler.schedule(
        project=current_app.config.get('CRAWLER_PROJECT'),
        spider=spider,
        settings=crawler_settings,
        **crawler_arguments
    )
    if job_id:
        crawler_job = CrawlerJob.create(
            job_id=job_id,
            spider=spider,
            workflow=workflow,
        )
        db.session.commit()
        current_app.logger.info(
            "Scheduled scrapyd job with id: {0}".format(job_id)
        )
        current_app.logger.info(
            "Created crawler job with id:{0}".format(crawler_job.id)
        )
    else:
        raise CrawlerScheduleError(
            "Could not schedule '{0}' spider for project '{1}'".format(
                spider, current_app.config.get('CRAWLER_PROJECT')
            )
        )

    return str(crawler_job.job_id)
42930
public void perform() {
        for (int i = 0; i < operations.size(); i++) {
            operations.get(i).process();
        }
    }
1156866
def n_executions(self):
        
        pipeline = self.tiger.connection.pipeline()
        pipeline.exists(self.tiger._key('task', self.id))
        pipeline.llen(self.tiger._key('task', self.id, 'executions'))
        exists, n_executions = pipeline.execute()
        if not exists:
            raise TaskNotFound('Task {} not found.'.format(
                self.id
            ))
        return n_executions
558383
public function crawl()
    {
        $this->getQueueManager()->addUri($this->seed);
        $this->getDownloader()->getPersistenceHandler()->setSpiderId($this->spiderId);

        $this->doCrawl();
    }
850231
def status
      total = self.crawl_segments.count
      remaining = CrawlSegment.where(:crawl_id => self.id,
                                        :parse_time => nil).count
      parsed = total - remaining
      status = self.crawl_name
      status += " Segments: to parse #{remaining}, "
      status += "parsed #{parsed}, total #{total}"
    end
209687
@Override
    protected void doPerform() {
        WebElement targetElem = getFirstElement(target);
        if (targetOffset == null) {
            getActions().dragAndDrop(getFirstElement(), targetElem).perform();
        } else {
            org.openqa.selenium.Dimension size = targetElem.getSize();
            Dimension dimension = new Dimension(size.width, size.height);
            Point offsetPoint = targetOffset.offset(dimension);
            getActions().dragAndDropBy(getFirstElement(), offsetPoint.x(), offsetPoint.y()).perform();
        }
    }
561859
@Override
	public CrawlSession call() {
		setMaximumCrawlTimeIfNeeded();
		plugins.runPreCrawlingPlugins(config);
		CrawlTaskConsumer firstConsumer = consumerFactory.get();
		StateVertex firstState = firstConsumer.crawlIndex();
		crawlSessionProvider.setup(firstState);
		plugins.runOnNewStatePlugins(firstConsumer.getContext(), firstState);
		executeConsumers(firstConsumer);
		return crawlSessionProvider.get();
	}
396662
function crawl(type, handler) {
		gui.Object.each(type, istype).forEach(
			function(type) {
				handle(type, handler);
				crawl(type, handler);
			}
		);
	}
812215
def total_write_throughput(self):
        
        total = self.write_throughput
        for index in itervalues(self.global_indexes):
            total += index.write_throughput
        return total
76743
def report_self(self):
        '''
        
        '''
        self.logger.debug("Reporting self id", extra={'uuid':self.my_uuid})
        key = "stats:crawler:{m}:{s}:{u}".format(
            m=socket.gethostname(),
            s=self.spider.name,
            u=self.my_uuid)
        self.redis_conn.set(key, time.time())
        self.redis_conn.expire(key, self.ip_update_interval * 2)
1173435
def list_crawler_workflows(tail):
    """"""
    query = (
        models.CrawlerWorkflowObject.query
        .order_by(models.CrawlerWorkflowObject.object_id.desc())
    )
    if tail != 0:
        query = query.limit(tail)

    workflows = query.yield_per(10).all()
    _show_table(results=workflows)
176669
def crawler(site, uri=None):
    """"""
    config = load_site_config(site)
    model = _get_model('crawl', config, uri)
    visited_set, visited_uri_set, consume_set, crawl_set = get_site_sets(
        site, config
    )

    if not visited_set.has(model.hash):
        visited_set.add(model.hash)
        visited_uri_set.add(model.uri)

        if (
            model.is_consume_page
            and not consume_set.has(model.hash)
        ):
            consume_set.add(model.hash)
            consume_q.enqueue(
                consumer,
                site,
                model.uri
            )
        else:
            for crawl_uri in model.uris_to_crawl:
                if (
                    not visited_uri_set.has(crawl_uri)
                    and not crawl_set.has(crawl_uri)
                ):
                    crawl_set.add(crawl_uri)
                    crawl_q.enqueue(
                        crawler,
                        site,
                        crawl_uri
                    )
1303987
private void performDelay() {
        try {
            TimeUnit.MILLISECONDS.sleep(crawlDelayMechanism.getDelay());
        } catch (InterruptedException ex) {
            Thread.currentThread().interrupt();
            isStopping = true;
        }
    }
1338974
def size(cls, crawler):
        """"""
        key = make_key('queue_pending', crawler)
        return unpack_int(conn.get(key))
913322
def total_read_throughput(self):
        
        total = self.read_throughput
        for index in itervalues(self.global_indexes):
            total += index.read_throughput
        return total
335521
def _run(self, url, auth):
        '''
        
        '''
        if url in self.visited:
            logger.debug("Skipping %s (already crawled)" % url)
            return
        self.visited.append(url)

        logger.info("Crawling: %s" % url)
        url = self._get_catalog_url(url)

        # Get an etree object
        xml_content = request_xml(url, auth)
        for ds in self._build_catalog(url, xml_content):
            yield ds