kafka download and install
        :return:
6188
def kafka_install(self):
        
        with cd('/tmp'):
            if not exists('kafka.tgz'):
                sudo('wget {0} -O kafka.tgz'.format(
                    bigdata_conf.kafka_download_url
                ))

            sudo('tar -zxf kafka.tgz')

            sudo('rm -rf {0}'.format(bigdata_conf.kafka_home))
            sudo('mv kafka_* {0}'.format(bigdata_conf.kafka_home))
574451
def get_rest_api():
    
    assert _has_local_install

    url=[]
    ok = _run_st(['geturl', '--api'], lines=url)
    if not ok:
        raise ChildProcessError('streamtool geturl')
    return url[0]
203900
def install_from_zip(pkgpath, install_path, register_func, delete_after_install=False):
    """"""
    logger.debug("%s is a file, attempting to load zip", pkgpath)
    pkgtempdir = tempfile.mkdtemp(prefix="honeycomb_")
    try:
        with zipfile.ZipFile(pkgpath) as pkgzip:
            pkgzip.extractall(pkgtempdir)
    except zipfile.BadZipfile as exc:
        logger.debug(str(exc))
        raise click.ClickException(str(exc))
    if delete_after_install:
        logger.debug("deleting %s", pkgpath)
        os.remove(pkgpath)
    logger.debug("installing from unzipped folder %s", pkgtempdir)
    return install_dir(pkgtempdir, install_path, register_func, delete_after_install=True)
267354
def download_series_gui(frame, urls, directory, min_file_size, max_file_size, no_redirects):
	

	# create directory to save files
	if not os.path.exists(directory):
		os.makedirs(directory)
	app = progress_class(frame, urls, directory, min_file_size, max_file_size, no_redirects)
481679
def download(self, path=None, show_progress=True, resume=True,
                 auto_retry=True, proapi=False):
        """"""
        self.api.download(self, path, show_progress, resume, auto_retry,
                          proapi)
752221
def download(model=None, direct=False, *pip_args):
    
    if model is None:
        model = about.__default_corpus__

    if direct:
        dl = download_model('{m}/{m}.tar.gz#egg={m}'.format(m=model), pip_args)
    else:
        shortcuts = get_json(about.__shortcuts__, "available shortcuts")
        model_name = shortcuts.get(model, model)
        compatibility = get_compatibility()
        version = get_version(model_name, compatibility)
        dl = download_model('{m}-{v}/{m}-{v}.tar.gz#egg={m}=={v}'
                            .format(m=model_name, v=version), pip_args)
        if dl != 0:  # if download subprocess doesn't return 0, exit
            sys.exit(dl)
        try:
            # Get package path here because link uses
            # pip.get_installed_distributions() to check if model is a
            # package, which fails if model was just installed via
            # subprocess
            package_path = get_package_path(model_name)
            link(model_name, model, force=True, model_path=package_path)
        except:
            # Dirty, but since spacy.download and the auto-linking is
            # mostly a convenience wrapper, it's best to show a success
            # message and loading instructions, even if linking fails.
            prints(Messages.M001.format(name=model_name), title=Messages.M002)
1182000
def _setup_kafka(self):
        
        # close older connections
        if self.consumer is not None:
            self.logger.debug("Closing existing kafka consumer")
            self.consumer.close()
            self.consumer = None
        if self.producer is not None:
            self.logger.debug("Closing existing kafka producer")
            self.producer.flush()
            self.producer.close(timeout=10)
            self.producer = None

        # create new connections
        self._consumer_thread = None
        self.logger.debug("Creating kafka connections")
        self.consumer = self._create_consumer()
        if not self.closed:
            self.logger.debug("Kafka Conumer created")
        self.producer = self._create_producer()
        if not self.closed:
            self.logger.debug("Kafka Producer created")

        if not self.closed:
            self.kafka_connected = True
            self.logger.info("Connected successfully to Kafka")
            self._spawn_kafka_consumer_thread()
848121
def acquire(self, **kwargs):
        
        return config.download_data(self.temp_filename, self.url,
                                    self.sha256)
792771
def _get_kafka_consumer_offsets(self, instance, consumer_groups):
        
        consumer_offsets = {}
        topics = defaultdict(set)

        cli = self._get_kafka_client(instance)

        for consumer_group, topic_partitions in iteritems(consumer_groups):
            try:
                coordinator_id = self._get_group_coordinator(cli, consumer_group)
                if coordinator_id:
                    offsets = self._get_consumer_offsets(cli, consumer_group, topic_partitions, coordinator_id)
                else:
                    offsets = self._get_consumer_offsets(cli, consumer_group, topic_partitions)
                    self.log.info("unable to find group coordinator for %s", consumer_group)

                for (topic, partition), offset in iteritems(offsets):
                    topics[topic].update([partition])
                    key = (consumer_group, topic, partition)
                    consumer_offsets[key] = offset
            except Exception:
                self.log.exception('Could not read consumer offsets from kafka.')

        return consumer_offsets, topics
1372199
def main():
    """"""
    options = _parse_args()
    archive = download_setuptools(
        version=options.version,
        download_base=options.download_base,
        downloader_factory=options.downloader_factory,
    )
    return _install(archive, _build_install_args(options))
501486
def init(self):
        """"""
        if not self.export_enable:
            return None

        # Build the server URI with host and port
        server_uri = '{}:{}'.format(self.host, self.port)

        try:
            s = KafkaProducer(bootstrap_servers=server_uri,
                              value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                              compression_type=self.compression)
        except Exception as e:
            logger.critical("Cannot connect to Kafka server %s (%s)" % (server_uri, e))
            sys.exit(2)
        else:
            logger.info("Connected to the Kafka server %s" % server_uri)

        return s
658347
function() {
    _cbt =  require('./cbt')();
    _options = options || {};

    // apply defaults
    // the min bytes of an incoming reply
    _options.minBytes = _options.minBytes || 1;

    // the max bytes of an incoming reply
    _options.maxBytes = _options.maxBytes || 1024 * 1024;

    // the group the client will join when using `.connect()`
    _options.group = _options.group || 'kafkaesqueGroup';

    // the clientID when connecting to kafka
    _options.clientId = _options.clientId || 'kafkaesque' + Math.floor(Math.random() * 100000000);

    // the array of brokers
    _options.brokers = _options.brokers || [{host: 'localhost', port: 9092}];

    // the amount of time it should take for this clients session within the group
    // to timeout
    _options.sessionTimeout = _options.sessionTimeout || 6000;

    // the amount of time to take to emit a heartbeat msg
    _options.heartbeat = _options.heartbeat || 2500;

    // the default amount of time that the kafka broker shoudl wait to send
    // a reply to a fetch request if the fetch reply is smaller than the minBytes
    _options.maxWait = _options.maxWait || 5000;

    _brokers = {};
    _topics = {};
    _groupMemberId = '';
    _groupGeneration = 0;
    _.each(_options.brokers, function(broker) {
      broker = {
        host: broker.host,
        port: broker.port,
        maxBytes: _options.maxBytes,
        minBytes: _options.minBytes,
        clientId: _options.clientId
      };

      _brokers[_makeBrokerKey(broker)] = api(broker);
    });

    _metaBroker = _brokers[_makeBrokerKey(_options.brokers[0])];
  }
1360675
def download(args):
    
    session = c.Session(args)
    issues = aux.parse_for_download(args)
    if issues == ['']:
        sys.exit(
            "You need to give a list of issues, of the form ""a, b-c, d...""")
    dumpfilename = os.path.join(session.data_dir, 'feeddump')
    if not os.path.isfile(dumpfilename):
        sys.exit(
            ("You need to run ""greg check"
             "<feed>"" before using ""greg download""."))
    with open(dumpfilename, mode='rb') as dumpfile:
        dump = pickle.load(dumpfile)
    try:
        feed = c.Feed(session, dump[0], dump[1])
    except Exception:
        sys.exit((
            "... something went wrong."
            "Are you sure your last ""greg check"" went well?"))
    for number in issues:
        entry = dump[1].entries[eval(number)]
        feed.info = []
        feed.entrylinks = []
        feed.fix_linkdate(entry)
        feed.download_entry(entry)
747689
def download(url, target, headers=None, trackers=()):
    
    if headers is None:
        headers = {}
    headers.setdefault('user-agent', 'requests_download/'+__version__)
    r = requests.get(url, headers=headers, stream=True)
    r.raise_for_status()
    for t in trackers:
        t.on_start(r)


    with open(target, 'wb') as f:
        for chunk in r.iter_content(chunk_size=8192):
            if chunk:
                f.write(chunk)
                for t in trackers:
                    t.on_chunk(chunk)

    for t in trackers:
        t.on_finish()
426033
def logstash_install(self):
        
        with cd('/tmp'):
            if not exists('logstash.deb'):
                sudo('wget {0} -O logstash.deb'.format(
                    bigdata_conf.logstash_download_url
                ))

            sudo('dpkg -i logstash.deb')
            sudo('apt-get install -y')
81073
def _sync_kaggle_download(self, kaggle_url, destination_path):
    """"""
    kaggle_file = kaggle.KaggleFile.from_url(kaggle_url)
    downloader = self.kaggle_downloader(kaggle_file.competition)
    filepath = downloader.download_file(kaggle_file.filename, destination_path)

    dl_size = tf.io.gfile.stat(filepath).length
    checksum = self._checksumer()
    with tf.io.gfile.GFile(filepath, 'rb') as f:
      while True:
        block = f.read(io.DEFAULT_BUFFER_SIZE)
        if not block:
          break
        checksum.update(block)
    return checksum.hexdigest(), dl_size
395530
def get_default_config(self):
        
        config = super(KafkaCollector, self).get_default_config()
        config.update({
            'host': '127.0.0.1',
            'port': 8082,
            'path': 'kafka',
        })
        return config