A powerful spider system in python.
2373
def cli(ctx, **kwargs):
    
    if kwargs['add_sys_path']:
        sys.path.append(os.getcwd())

    logging.config.fileConfig(kwargs['logging_config'])

    # get db from env
    for db in ('taskdb', 'projectdb', 'resultdb'):
        if kwargs[db] is not None:
            continue
        if os.environ.get('MYSQL_NAME'):
            kwargs[db] = utils.Get(lambda db=db: connect_database(
                'sqlalchemy+mysql+%s://%s:%s/%s' % (
                    db, os.environ['MYSQL_PORT_3306_TCP_ADDR'],
                    os.environ['MYSQL_PORT_3306_TCP_PORT'], db)))
        elif os.environ.get('MONGODB_NAME'):
            kwargs[db] = utils.Get(lambda db=db: connect_database(
                'mongodb+%s://%s:%s/%s' % (
                    db, os.environ['MONGODB_PORT_27017_TCP_ADDR'],
                    os.environ['MONGODB_PORT_27017_TCP_PORT'], db)))
        elif ctx.invoked_subcommand == 'bench':
            if kwargs['data_path'] == './data':
                kwargs['data_path'] += '/bench'
                shutil.rmtree(kwargs['data_path'], ignore_errors=True)
                os.mkdir(kwargs['data_path'])
            if db in ('taskdb', 'resultdb'):
                kwargs[db] = utils.Get(lambda db=db: connect_database('sqlite+%s://' % (db)))
            elif db in ('projectdb', ):
                kwargs[db] = utils.Get(lambda db=db: connect_database('local+%s://%s' % (
                    db, os.path.join(os.path.dirname(__file__), 'libs/bench.py'))))
        else:
            if not os.path.exists(kwargs['data_path']):
                os.mkdir(kwargs['data_path'])
            kwargs[db] = utils.Get(lambda db=db: connect_database('sqlite+%s:///%s/%s.db' % (
                db, kwargs['data_path'], db[:-2])))
            kwargs['is_%s_default' % db] = True

    # create folder for counter.dump
    if not os.path.exists(kwargs['data_path']):
        os.mkdir(kwargs['data_path'])

    # message queue, compatible with old version
    if kwargs.get('message_queue'):
        pass
    elif kwargs.get('amqp_url'):
        kwargs['message_queue'] = kwargs['amqp_url']
    elif os.environ.get('RABBITMQ_NAME'):
        kwargs['message_queue'] = ("amqp://guest:guest@%(RABBITMQ_PORT_5672_TCP_ADDR)s"
                                   ":%(RABBITMQ_PORT_5672_TCP_PORT)s/%%2F" % os.environ)
    elif kwargs.get('beanstalk'):
        kwargs['message_queue'] = "beanstalk://%s/" % kwargs['beanstalk']

    for name in ('newtask_queue', 'status_queue', 'scheduler2fetcher',
                 'fetcher2processor', 'processor2result'):
        if kwargs.get('message_queue'):
            kwargs[name] = utils.Get(lambda name=name: connect_message_queue(
                name, kwargs.get('message_queue'), kwargs['queue_maxsize']))
        else:
            kwargs[name] = connect_message_queue(name, kwargs.get('message_queue'),
                                                 kwargs['queue_maxsize'])

    # phantomjs-proxy
    if kwargs.get('phantomjs_proxy'):
        pass
    elif os.environ.get('PHANTOMJS_NAME'):
        kwargs['phantomjs_proxy'] = os.environ['PHANTOMJS_PORT_25555_TCP'][len('tcp://'):]

    # puppeteer-proxy
    if kwargs.get('puppeteer_proxy'):
        pass
    elif os.environ.get('PUPPETEER_NAME'):
        kwargs['puppeteer_proxy'] = os.environ['PUPPETEER_PORT_22222_TCP'][len('tcp://'):]

    ctx.obj = utils.ObjectDict(ctx.obj or {})
    ctx.obj['instances'] = []
    ctx.obj.update(kwargs)

    if ctx.invoked_subcommand is None and not ctx.obj.get('testing_mode'):
        ctx.invoke(all)
    return ctx
1326624
def killer(site):
    """"""
    config = load_site_config(site)
    crawl_q.empty()
    consume_q.empty()

    for site_set in get_site_sets(site, config):
        site_set.destroy()
246126
def scrape(cls, selector, root, xpath=False):
        """"""
        log.debug('Called scrape classmethod with root: %s' % root)
        roots = selector.xpath(root) if xpath else selector.css(root)
        results = [cls(r) for r in roots]
        return EntityList(*results)
626169
def crawl(self):
        """"""
        page = Linkfetcher(self.root)
        page.linkfetch()
        queue = Queue()
        for url in page.urls:
            queue.put(url)
        followed = [self.root]

        n = 0

        while True:
            try:
                url = queue.get()
            except QueueEmpty:
                break

            n += 1

            if url not in followed:
                try:

                    host = urllib.parse.urlparse(url)[1]

                    if self.locked and re.match(".*%s" % self.host, host):
                        followed.append(url)
                        self.followed += 1
                        page = Linkfetcher(url)
                        page.linkfetch()
                        for i, url in enumerate(page):
                            if url not in self.urls:
                                self.links += 1
                                queue.put(url)
                                self.urls.append(url)

                        if n > self.depth and self.depth > 0:
                            break
                except Exception as e:
                    print("ERROR: The URL '%s' can't be crawled (%s)" % (url, e))
                    print(format_exc())
1146635
public void crawlToOpen(TreePath path, ArrayList<BugLeafNode> bugLeafNodes, ArrayList<TreePath> treePaths) {
        for (int i = 0; i < getChildCount(path.getLastPathComponent()); i++) {
            if (!isLeaf(getChild(path.getLastPathComponent(), i))) {
                for (BugLeafNode p : bugLeafNodes) {
                    if (p.matches((BugAspects) getChild(path.getLastPathComponent(), i))) {
                        tree.expandPath(path);
                        crawlToOpen(path.pathByAddingChild(getChild(path.getLastPathComponent(), i)), bugLeafNodes, treePaths);
                        break;
                    }
                }
            } else {
                for (BugLeafNode b : bugLeafNodes) {
                    if (getChild(path.getLastPathComponent(), i).equals(b)) {
                        tree.expandPath(path);
                        treePaths.add(path.pathByAddingChild(getChild(path.getLastPathComponent(), i)));
                    }
                }
            }
        }
    }
467731
def pquery(self, x_list, k=1, eps=0, p=2,
               distance_upper_bound=np.inf):
        
        x = np.array(x_list)
        nx, mx = x.shape
        shmem_x = mp.Array(ctypes.c_double, nx*mx)
        shmem_d = mp.Array(ctypes.c_double, nx*k)
        shmem_i = mp.Array(ctypes.c_double, nx*k)

        _x = shmem_as_nparray(shmem_x).reshape((nx, mx))
        _d = shmem_as_nparray(shmem_d).reshape((nx, k))

        _i = shmem_as_nparray(shmem_i)
        if k != 1:
            _i = _i.reshape((nx, k))

        _x[:, :] = x

        nprocs = num_cpus()
        scheduler = Scheduler(nx, nprocs)

        ierr = mp.Value(ctypes.c_int, 0)

        query_args = (scheduler,
                      self.shmem_data, self.n, self.m, self.leafsize,
                      shmem_x, nx, shmem_d, shmem_i,
                      k, eps, p, distance_upper_bound,
                      ierr)
        pool = [mp.Process(target=_pquery, args=query_args) for _ in range(nprocs)]
        for p in pool: p.start()
        for p in pool: p.join()
        if ierr.value != 0:
            raise RuntimeError('%d errors in worker processes' % (ierr.value))

        return _d.copy(), _i.astype(int).copy()
849358
def schedule_next_request(self):
        
        req = self.next_request()

        if req:
            self.crawler.engine.crawl(req, spider=self)
920647
function Crawler(server, conf, logger) {

  stream.Transform.call(this, { objectMode: true });

  const peerCrawler = new PeerCrawler(server, conf, logger);
  const peerTester = new PeerTester(server, conf, logger);
  const blockCrawler = new BlockCrawler(server, logger, this);
  const sandboxCrawler = new SandboxCrawler(server, conf, logger);

  this.pullBlocks = blockCrawler.pullBlocks;
  this.sandboxPull = sandboxCrawler.sandboxPull;

  this.startService = () => co(function*() {
    return yield [
      peerCrawler.startService(),
      peerTester.startService(),
      blockCrawler.startService(),
      sandboxCrawler.startService()
    ];
  });

  this.stopService = () => co(function*() {
    return yield [
      peerCrawler.stopService(),
      peerTester.stopService(),
      blockCrawler.stopService(),
      sandboxCrawler.stopService()
    ];
  });
}
968350
def process_spider_input(self, response, spider):
        '''
        
        '''
        self.logger.debug("processing redis stats middleware")
        if self.settings['STATS_STATUS_CODES']:
            if spider.name not in self.stats_dict:
                self._setup_stats_status_codes(spider.name)

            if 'status_codes' in self.stats_dict[spider.name]:
                code = response.status
                if code in self.settings['STATS_RESPONSE_CODES']:
                    for key in self.stats_dict[spider.name]['status_codes'][code]:
                        try:
                            if key == 'lifetime':
                                unique = response.url + str(response.status)\
                                    + str(time.time())
                                self.stats_dict[spider.name]['status_codes'][code][key].increment(unique)
                            else:
                                self.stats_dict[spider.name]['status_codes'][code][key].increment()
                        except Exception as e:
                            self.logger.warn("Error in spider redis stats")
                    self.logger.debug("Incremented status_code '{c}' stats"\
                            .format(c=code))
1306040
def manage_crawlers(self):
        
        sites = self.json.get_site_objects()
        for index, site in enumerate(sites):
            if "daemonize" in site:
                self.daemon_list.add_daemon(index, site["daemonize"])
            elif "additional_rss_daemon" in site:
                self.daemon_list.add_daemon(index,
                                            site["additional_rss_daemon"])
                self.crawler_list.append_item(index)
            else:
                self.crawler_list.append_item(index)

        num_threads = self.cfg.section('Crawler')[
            'number_of_parallel_crawlers']
        if self.crawler_list.len() < num_threads:
            num_threads = self.crawler_list.len()

        for _ in range(num_threads):
            thread = threading.Thread(target=self.manage_crawler,
                                      args=(),
                                      kwargs={})
            self.threads.append(thread)
            thread.start()

        num_daemons = self.cfg.section('Crawler')['number_of_parallel_daemons']
        if self.daemon_list.len() < num_daemons:
            num_daemons = self.daemon_list.len()

        for _ in range(num_daemons):
            thread_daemonized = threading.Thread(target=self.manage_daemon,
                                                 args=(),
                                                 kwargs={})
            self.threads_daemonized.append(thread_daemonized)
            thread_daemonized.start()

        while not self.shutdown:
            try:
                time.sleep(10)
                # if we are not in daemon mode and no crawler is running any longer,
                # all articles have been crawled and the tool can shut down
                if self.daemon_list.len() == 0 and self.number_of_active_crawlers == 0:
                    self.graceful_stop()
                    break

            except IOError:
                # This exception will only occur on kill-process on windows.
                # The process should be killed, thus this exception is
                # irrelevant.
                pass
661265
public static void main(String[] args) throws IOException {
		CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(URL);
		
		builder.crawlRules().setFormFillMode(FormFillMode.RANDOM);

		// click these elements
		builder.crawlRules().clickDefaultElements();
		 /*builder.crawlRules().click("A");
		 builder.crawlRules().click("button");*/
		builder.crawlRules().crawlHiddenAnchors(true);
		builder.crawlRules().crawlFrames(false);
		builder.setUnlimitedCrawlDepth();
		builder.setUnlimitedRuntime();
		builder.setUnlimitedStates();

		//builder.setMaximumStates(10);
		//builder.setMaximumDepth(3);
		builder.crawlRules().clickElementsInRandomOrder(false);

		// Set timeouts
		builder.crawlRules().waitAfterReloadUrl(WAIT_TIME_AFTER_RELOAD, TimeUnit.MILLISECONDS);
		builder.crawlRules().waitAfterEvent(WAIT_TIME_AFTER_EVENT, TimeUnit.MILLISECONDS);

		builder.setBrowserConfig(new BrowserConfiguration(BrowserType.PHANTOMJS, 1));

		// CrawlOverview
		builder.addPlugin(new CrawlOverview());

		CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
		crawljax.call();

	}
601411
def option_parser():
    """"""
    usage = '''
             $ ./crawler -d5 <url>
                Here in this case it goes till depth of 5 and url is target URL to
                start crawling.
            '''
    version = "2.0.0"

    parser = optparse.OptionParser(usage=usage, version=version)

    parser.add_option("-l", "--links", action="store_true",
                      default=False, dest="links", help="links for target url")

    parser.add_option("-d", "--depth", action="store", type="int",
                      default=30, dest="depth", help="Maximum depth traverse")
    opts, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        raise SystemExit(1)

    return opts, args
1274615
def crawl_up(arg):
    # type: (str) -> Tuple[str, str]
    
    dir, mod = os.path.split(arg)
    mod = strip_py(mod) or mod
    while dir and get_init_file(dir):
        dir, base = os.path.split(dir)
        if not base:
            break
        if mod == '__init__' or not mod:
            mod = base
        else:
            mod = base + '.' + mod
    return dir, mod
219927
public static void main(String[] args) {

    try {
      final CrawlToFile crawl = new CrawlToFile(args);
      crawl.crawl();

    } catch (ParseException e) {
      System.err.print(e.getMessage());
    } catch (IllegalArgumentException e) {
      System.err.println(e.getMessage());
    }

  }
1187634
def crawl(self, urls, name='crawl', api='analyze', **kwargs):
        
        # If multiple seed URLs are specified, join with whitespace.
        if isinstance(urls, list):
            urls = ' '.join(urls)
        url = self.endpoint('crawl')
        process_url = self.endpoint(api)
        params = {
            'token': self._token,
            'seeds': urls,
            'name': name,
            'apiUrl': process_url,
        }

        # Add any additional named parameters as accepted by Crawlbot
        params['maxToCrawl'] = 10
        params.update(kwargs)

        self._get(url, params=params)

        return Job(self._token, name, self._version)
275559
function crawlScripts(result, $, endCallback) {

    if (! result.scripts) {
      return endCallback();
    }

    log.debug({"url" : result.url, "step" : "analyzeHTML", "message" : "CrawlScripts"});

    async.each($('script'), function(script, callback) {
        crawlScript($, result, script, callback);
    }, endCallback);
}
839431
function Scraper(url,window,router){
	this.url = url;
	this.window = window;
	this.router = router;
}