Generate a list of all crawlers, alphabetically, with op counts.
176130
def index():
    """"""
    crawlers = []
    for crawler in manager:
        data = Event.get_counts(crawler)
        data['last_active'] = crawler.last_run
        data['total_ops'] = crawler.op_count
        data['running'] = crawler.is_running
        data['crawler'] = crawler
        crawlers.append(data)
    return render_template('index.html', crawlers=crawlers)
1312495
def _build_crawlid_info(self, master, dict):
        '''
        
        '''
        master['total_pending'] = 0
        master['total_domains'] = 0
        master['appid'] = dict['appid']
        master['crawlid'] = dict['crawlid']
        master['spiderid'] = dict['spiderid']
        master['domains'] = {}

        timeout_key = 'timeout:{sid}:{aid}:{cid}'.format(sid=dict['spiderid'],
                                                         aid=dict['appid'],
                                                         cid=dict['crawlid'])
        if self.redis_conn.exists(timeout_key):
            master['expires'] = self.redis_conn.get(timeout_key)

        # get all domain queues
        match_string = '{sid}:*:queue'.format(sid=dict['spiderid'])
        for key in self.redis_conn.scan_iter(match=match_string):
            domain = key.split(":")[1]
            sortedDict = self._get_bin(key)

            # now iterate through binned dict
            for score in sortedDict:
                for item in sortedDict[score]:
                    if 'meta' in item:
                        item = item['meta']
                    if item['appid'] == dict['appid'] and item['crawlid'] == dict['crawlid']:
                        if domain not in master['domains']:
                            master['domains'][domain] = {}
                            master['domains'][domain]['total'] = 0
                            master['domains'][domain]['high_priority'] = -9999
                            master['domains'][domain]['low_priority'] = 9999
                            master['total_domains'] = master['total_domains'] + 1

                        master['domains'][domain]['total'] = master['domains'][domain]['total'] + 1

                        if item['priority'] > master['domains'][domain]['high_priority']:
                            master['domains'][domain]['high_priority'] = item['priority']

                        if item['priority'] < master['domains'][domain]['low_priority']:
                            master['domains'][domain]['low_priority'] = item['priority']

                        master['total_pending'] = master['total_pending'] + 1

        return master
720215
def list_xz (archive, compression, cmd, verbosity, interactive):
    """"""
    cmdlist = [cmd]
    cmdlist.append('-l')
    if verbosity > 1:
        cmdlist.append('-v')
    cmdlist.append(archive)
    return cmdlist
11305
def list_by_cid(self):
        
        prs = self._processors
        return sorted(((cid, [prs[cid]]) for cid in sorted(prs.keys())),
                      key=operator.itemgetter(0))
645100
def list():
    
    functions = lambder.list_functions()
    output = json.dumps(
        functions,
        sort_keys=True,
        indent=4,
        separators=(',', ':')
    )
    click.echo(output)
1199343
def gen_list_name():
    
    for name in _LIST_NAMES:
        yield name

    counter = itertools.count(1000)
    while True:
        yield b(str(next(counter))) + b"\x00"
1259999
def list(self, limit=None, marker=None, name=None, visibility=None,
            member_status=None, owner=None, tag=None, status=None,
            size_min=None, size_max=None, sort_key=None, sort_dir=None,
            return_raw=False):
        
        uri = "/%s" % self.uri_base
        qs = utils.dict_to_qs(dict(limit=limit, marker=marker, name=name,
                visibility=visibility, member_status=member_status,
                owner=owner, tag=tag, status=status, size_min=size_min,
                size_max=size_max, sort_key=sort_key, sort_dir=sort_dir))
        if qs:
            uri = "%s?%s" % (uri, qs)
        return self._list(uri, return_raw=return_raw)
263973
def get_all_sources(self):
        """"""
        result = []
        for batch in self.batches:
            result.extend(batch.sources)
        return result
584650
def list_jobs(tail):
    """"""
    query = (
        db.session.query(models.CrawlerJob)
        .order_by(models.CrawlerJob.id.desc())
    )
    if tail != 0:
        query = query.limit(tail)

    results = query.yield_per(10).all()
    _show_table(results=results)
553721
def list(self, limit=None, marker=None, name=None, visibility=None,
            member_status=None, owner=None, tag=None, status=None,
            size_min=None, size_max=None, sort_key=None, sort_dir=None):
        
        return self._manager.list(limit=limit, marker=marker, name=name,
                visibility=visibility, member_status=member_status,
                owner=owner, tag=tag, status=status, size_min=size_min,
                size_max=size_max, sort_key=sort_key, sort_dir=sort_dir)
772695
def list(self, limit=None, marker=None, end_marker=None, prefix=None):
        
        return self._manager.list(limit=limit, marker=marker,
                end_marker=end_marker, prefix=prefix)
1306040
def manage_crawlers(self):
        
        sites = self.json.get_site_objects()
        for index, site in enumerate(sites):
            if "daemonize" in site:
                self.daemon_list.add_daemon(index, site["daemonize"])
            elif "additional_rss_daemon" in site:
                self.daemon_list.add_daemon(index,
                                            site["additional_rss_daemon"])
                self.crawler_list.append_item(index)
            else:
                self.crawler_list.append_item(index)

        num_threads = self.cfg.section('Crawler')[
            'number_of_parallel_crawlers']
        if self.crawler_list.len() < num_threads:
            num_threads = self.crawler_list.len()

        for _ in range(num_threads):
            thread = threading.Thread(target=self.manage_crawler,
                                      args=(),
                                      kwargs={})
            self.threads.append(thread)
            thread.start()

        num_daemons = self.cfg.section('Crawler')['number_of_parallel_daemons']
        if self.daemon_list.len() < num_daemons:
            num_daemons = self.daemon_list.len()

        for _ in range(num_daemons):
            thread_daemonized = threading.Thread(target=self.manage_daemon,
                                                 args=(),
                                                 kwargs={})
            self.threads_daemonized.append(thread_daemonized)
            thread_daemonized.start()

        while not self.shutdown:
            try:
                time.sleep(10)
                # if we are not in daemon mode and no crawler is running any longer,
                # all articles have been crawled and the tool can shut down
                if self.daemon_list.len() == 0 and self.number_of_active_crawlers == 0:
                    self.graceful_stop()
                    break

            except IOError:
                # This exception will only occur on kill-process on windows.
                # The process should be killed, thus this exception is
                # irrelevant.
                pass
748839
function crawlHrefs(result, $, endCallback) {

    log.debug({"url" : result.url, "step" : "analyzeHTML", "message" : "CrawlHrefs"});
    async.each($('a'), function(a, callback) {
        crawlHref($, result, a, callback);
    }, endCallback);

}
408603
def page_list(cls, pages, base_url='/'):
        """"""
        plist = []
        for page in pages:
            url = "<a href=\"{}{}\">{}</a>".format(base_url, cls.slugify(page), page)
            plist.append(url)
        return plist
497039
def list_running():
    '''
    
    '''
    ret = []
    for line in _machinectl('list')['stdout'].splitlines():
        try:
            ret.append(line.split()[0])
        except IndexError:
            pass
    return sorted(ret)
64836
def get_all(self):
    """"""
    ret = []
    for cache_name, stat in self.stats_per_cache.items():
      ret.append({
        'cache_name': cache_name,
        'num_hits': len(stat.hit_targets),
        'num_misses': len(stat.miss_targets),
        'hits': stat.hit_targets,
        'misses': stat.miss_targets
      })
    return ret
738487
def _get_all_scanner_ids(zap_helper):
    """"""
    scanners = zap_helper.zap.ascan.scanners()
    return [s['id'] for s in scanners]