Main driver that downloads, extracts, and deploys the oozie sharelib
342480
public void createShareLib() {

        if (!oozieShareLibCreate) {
            LOG.info("OOZIE: Share Lib Create Disabled... skipping");
        } else {

            final String fullOozieTarFilePath = shareLibCacheDir + Path.SEPARATOR + getOozieTarFileName();

            try {

                // Get and extract the oozie release
                getOozieTarFileFromRepo();
                String oozieExtractTempDir = extractOozieTarFileToTempDir(new File(fullOozieTarFilePath));

                // Extract the sharelib tarball to a temp dir
                String fullOozieShareLibTarFilePath = oozieExtractTempDir + Path.SEPARATOR +
                        "oozie-" + getOozieVersionFromOozieTarFileName() + Path.SEPARATOR +
                        "oozie-sharelib-" + getOozieVersionFromOozieTarFileName() + ".tar.gz";
                String oozieShareLibExtractTempDir = extractOozieShareLibTarFileToTempDir(
                        new File(fullOozieShareLibTarFilePath));

                // Copy the sharelib into HDFS
                Path destPath = new Path(oozieHdfsShareLibDir + Path.SEPARATOR + "oozie" + Path.SEPARATOR +
                        SHARE_LIB_PREFIX + getTimestampDirectory());
                LOG.info("OOZIE: Writing share lib contents to: {}", destPath);
                hdfsFileSystem.copyFromLocalFile(false, new Path(new File(oozieShareLibExtractTempDir).toURI()), destPath);

                if (purgeLocalShareLibCache) {
                    FileUtils.deleteDirectory(new File(shareLibCacheDir));
                }

            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
1267092
function downloadAndGenerate (template, cb) {
  var spinner = ora('downloading template')
  spinner.start()
  download(template, tmp, { clone: clone }, function (err) {
    spinner.stop()
    if (err) logger.fatal('Failed to download repo ' + template + ': ' + err.message.trim())
    generate(name, tmp, to, cb)
  })
}
637989
def download_extract(url):
    """"""
    logger.info("Downloading %s", url)
    request = urllib2.Request(url)
    request.add_header('User-Agent',
                       'caelum/0.1 +https://github.com/nrcharles/caelum')
    opener = urllib2.build_opener()
    with tempfile.TemporaryFile(suffix='.zip', dir=env.WEATHER_DATA_PATH) \
            as local_file:
        logger.debug('Saving to temporary file %s', local_file.name)
        local_file.write(opener.open(request).read())
        compressed_file = zipfile.ZipFile(local_file, 'r')
        logger.debug('Extracting %s', compressed_file)
        compressed_file.extractall(env.WEATHER_DATA_PATH)
        local_file.close()
1217402
def get_driver(secret_key=config.DEFAULT_SECRET_KEY, userid=config.DEFAULT_USERID,
               provider=config.DEFAULT_PROVIDER):

    """"""

    if hasattr(config, 'get_driver'):
        logger.debug('get_driver %s' % config.get_driver)
        return config.get_driver()
    else:
        logger.debug('get_driver {0}@{1}'.format(userid, provider))
        return libcloud.compute.providers.get_driver(
            config.PROVIDERS[provider])(userid, secret_key)
1159031
Driver provideHiveDriver() {
    HiveConf hiveConf = provideHiveConf();
    SessionState.start(hiveConf);

    return new Driver(hiveConf);
  }
1157818
def create_app(name, engine):
    
    try:
        if engine.lower() == "sqlalchemy":
            url = urlopen(SQLA_REPO_URL)
            dirname = "Flask-AppBuilder-Skeleton-master"
        elif engine.lower() == "mongoengine":
            url = urlopen(MONGOENGIE_REPO_URL)
            dirname = "Flask-AppBuilder-Skeleton-me-master"
        zipfile = ZipFile(BytesIO(url.read()))
        zipfile.extractall()
        os.rename(dirname, name)
        click.echo(click.style("Downloaded the skeleton app, good coding!", fg="green"))
        return True
    except Exception as e:
        click.echo(click.style("Something went wrong {0}".format(e), fg="red"))
        if engine.lower() == "sqlalchemy":
            click.echo(
                click.style(
                    "Try downloading from {0}".format(SQLA_REPO_URL), fg="green"
                )
            )
        elif engine.lower() == "mongoengine":
            click.echo(
                click.style(
                    "Try downloading from {0}".format(MONGOENGIE_REPO_URL), fg="green"
                )
            )
        return False
1372199
def main():
    """"""
    options = _parse_args()
    archive = download_setuptools(
        version=options.version,
        download_base=options.download_base,
        downloader_factory=options.downloader_factory,
    )
    return _install(archive, _build_install_args(options))
1013073
def main():
    
    # vars
    tar_file = TMPDIR + '/' + BINARY_URL.split('/')[-1]
    chksum = TMPDIR + '/' + MD5_URL.split('/')[-1]
    # pre-run validation + execution
    if precheck() and os_packages(distro.linux_distribution()):
        stdout_message('begin download')
        download()
        stdout_message('begin valid_checksum')
        valid_checksum(tar_file, chksum)
        return compile_binary(tar_file)
    logger.warning('%s: Pre-run dependency check fail - Exit' % inspect.stack()[0][3])
    return False
1345017
def init_driver(client_id):
    

    # Create profile directory if it does not exist
    profile_path = CHROME_CACHE_PATH + str(client_id)
    if not os.path.exists(profile_path):
        os.makedirs(profile_path)
    
    # Options to customize chrome window
    chrome_options = [
        'window-size=' + CHROME_WINDOW_SIZE,
        '--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/60.0.3112.78 Chrome/60.0.3112.78 Safari/537.36'
    ]
    if CHROME_IS_HEADLESS:
        chrome_options.append('--headless')
    if CHROME_DISABLE_GPU:
        chrome_options.append('--disable-gpu')
    
    # Create a whatsapidriver object
    d = WhatsAPIDriver(
        username=client_id, 
        profile=profile_path, 
        client='chrome', 
        chrome_options=chrome_options
    )
    return d
1303311
def main():
    
    windows_libraries = list(pylink.Library.find_library_windows())
    latest_library = None
    for lib in windows_libraries:
        if os.path.dirname(lib).endswith('JLinkARM'):
            # Always use the one pointed to by the 'JLinkARM' directory.
            latest_library = lib
            break
        elif latest_library is None:
            latest_library = lib
        elif os.path.dirname(lib) > os.path.dirname(latest_library):
            latest_library = lib

    if latest_library is None:
        raise OSError('No J-Link library found.')

    library = pylink.Library(latest_library)
    jlink = pylink.JLink(lib=library)

    print('Found version: %s' % jlink.version)

    for emu in jlink.connected_emulators():
        jlink.disable_dialog_boxes()
        jlink.open(serial_no=emu.SerialNumber)
        jlink.sync_firmware()
        print('Updated emulator with serial number %s' % emu.SerialNumber)

    return None
1363229
def _run(self):
        # type: (SyncCopy) -> None
        
        # mark start
        self._start_time = blobxfer.util.datetime_now()
        logger.info('blobxfer start time: {0}'.format(self._start_time))
        # initialize resume db if specified
        if self._general_options.resume_file is not None:
            self._resume = blobxfer.operations.resume.SyncCopyResumeManager(
                self._general_options.resume_file)
        # initialize download threads
        self._initialize_transfer_threads()
        # iterate through source paths to download
        processed_files = 0
        for src_ase, dst_ase in self._bind_sources_to_destination():
            processed_files += 1
            if self._general_options.dry_run:
                logger.info('[DRY RUN] synccopy: {} -> {}'.format(
                    src_ase.path, dst_ase.path))
            else:
                # add transfer to set
                with self._transfer_lock:
                    self._transfer_set.add(
                        blobxfer.operations.synccopy.SyncCopy.
                        create_unique_transfer_operation_id(src_ase, dst_ase))
                    self._synccopy_total += 1
                    self._synccopy_bytes_total += src_ase.size
                    if blobxfer.util.is_not_empty(dst_ase.replica_targets):
                        self._synccopy_bytes_total += (
                            len(dst_ase.replica_targets) * src_ase.size
                        )
                self._add_to_transfer_queue(src_ase, dst_ase)
        # set remote files processed
        with self._transfer_lock:
            self._all_remote_files_processed = True
            synccopy_size_mib = (
                self._synccopy_bytes_total / blobxfer.util.MEGABYTE
            )
            logger.debug(
                ('{0} remote files to sync, waiting for copy '
                 'completion of approx. {1:.4f} MiB').format(
                     processed_files, synccopy_size_mib))
        del processed_files
        # wait for downloads to complete
        self._wait_for_transfer_threads(terminate=False)
        end_time = blobxfer.util.datetime_now()
        # update progress bar
        self._update_progress_bar()
        # check for exceptions
        if len(self._exceptions) > 0:
            logger.error('exceptions encountered while downloading')
            # raise the first one
            raise self._exceptions[0]
        # check for mismatches
        if (self._synccopy_sofar != self._synccopy_total or
                self._synccopy_bytes_sofar != self._synccopy_bytes_total):
            raise RuntimeError(
                'copy mismatch: [count={}/{} bytes={}/{}]'.format(
                    self._synccopy_sofar, self._synccopy_total,
                    self._synccopy_bytes_sofar, self._synccopy_bytes_total))
        # delete all remaining local files not accounted for if
        # delete extraneous enabled
        self._delete_extraneous_files()
        # delete resume file if we've gotten this far
        if self._resume is not None:
            self._resume.delete()
        # output throughput
        if self._synccopy_start_time is not None:
            dltime = (end_time - self._synccopy_start_time).total_seconds()
            synccopy_size_mib = (
                (self._synccopy_bytes_total << 1) / blobxfer.util.MEGABYTE
            )
            dlmibspeed = synccopy_size_mib / dltime
            logger.info(
                ('elapsed copy time and throughput of {0:.4f} '
                 'GiB: {1:.3f} sec, {2:.4f} Mbps ({3:.3f} MiB/sec)').format(
                     synccopy_size_mib / 1024, dltime, dlmibspeed * 8,
                     dlmibspeed))
        end_time = blobxfer.util.datetime_now()
        logger.info('blobxfer end time: {0} (elapsed: {1:.3f} sec)'.format(
            end_time, (end_time - self._start_time).total_seconds()))
1016440
def rubocop_checksum
      ResultCache.source_checksum ||=
        begin
          lib_root = File.join(File.dirname(__FILE__), '..')
          exe_root = File.join(lib_root, '..', 'exe')

          # These are all the files we have `require`d plus everything in the
          # exe directory. A change to any of them could affect the cop output
          # so we include them in the cache hash.
          source_files = $LOADED_FEATURES + Find.find(exe_root).to_a
          sources = source_files
                    .select { |path| File.file?(path) }
                    .sort
                    .map { |path| IO.read(path, encoding: Encoding::UTF_8) }
          Digest::SHA1.hexdigest(sources.join)
        end
    end
418755
def _install_extension(self, addon, unpack=True):
        
        if addon == WEBDRIVER_EXT:
            addon = os.path.join(os.path.dirname(__file__), WEBDRIVER_EXT)

        tmpdir = None
        xpifile = None
        if addon.endswith('.xpi'):
            tmpdir = tempfile.mkdtemp(suffix='.' + os.path.split(addon)[-1])
            compressed_file = zipfile.ZipFile(addon, 'r')
            for name in compressed_file.namelist():
                if name.endswith('/'):
                    if not os.path.isdir(os.path.join(tmpdir, name)):
                        os.makedirs(os.path.join(tmpdir, name))
                else:
                    if not os.path.isdir(os.path.dirname(os.path.join(tmpdir, name))):
                        os.makedirs(os.path.dirname(os.path.join(tmpdir, name)))
                    data = compressed_file.read(name)
                    with open(os.path.join(tmpdir, name), 'wb') as f:
                        f.write(data)
            xpifile = addon
            addon = tmpdir

        # determine the addon id
        addon_details = self._addon_details(addon)
        addon_id = addon_details.get('id')
        assert addon_id, 'The addon id could not be found: %s' % addon

        # copy the addon to the profile
        addon_path = os.path.join(self.extensionsDir, addon_id)
        if not unpack and not addon_details['unpack'] and xpifile:
            if not os.path.exists(self.extensionsDir):
                os.makedirs(self.extensionsDir)
                os.chmod(self.extensionsDir, 0o755)
            shutil.copy(xpifile, addon_path + '.xpi')
        else:
            if not os.path.exists(addon_path):
                shutil.copytree(addon, addon_path, symlinks=True)

        # remove the temporary directory, if any
        if tmpdir:
            shutil.rmtree(tmpdir)
315554
def main():
    
    log = logging.getLogger(mod_logger + '.main')
    log.debug('This is DEBUG!')
    log.info('This is INFO!')
    log.warning('This is WARNING!')
    log.error('This is ERROR!')
    log.info('Running s3util.main...')
    my_bucket = 'cons3rt-deploying-cons3rt'
    my_regex = 'sourcebuilder.*apache-maven-.*3.3.3.*'
    try:
        s3util = S3Util(my_bucket)
    except S3UtilError as e:
        log.error('There was a problem creating S3Util:\n%s', e)
    else:
        log.info('Created S3Util successfully')
        key = s3util.find_key(my_regex)
        test = None
        if key is not None:
            test = s3util.download_file(key, '/Users/yennaco/Downloads')
        if test is not None:
            upload = s3util.upload_file(test, 'media-files-offline-assets/test')
            log.info('Upload result: %s', upload)
    log.info('End of main!')
582152
def download_run_and_upload(job, master_ip, inputs, spark_on_toil):
    
    master_ip = MasterAddress(master_ip)

    bam_name = inputs.sample.split('://')[-1].split('/')[-1]
    sample_name = ".".join(os.path.splitext(bam_name)[:-1])

    hdfs_subdir = sample_name + "-dir"

    if inputs.run_local:
        inputs.local_dir = job.fileStore.getLocalTempDir()
        if inputs.native_adam_path is None:
            hdfs_dir = "/data/"
        else:
            hdfs_dir = inputs.local_dir
    else:
        inputs.local_dir = None
        hdfs_dir = "hdfs://{0}:{1}/{2}".format(master_ip, HDFS_MASTER_PORT, hdfs_subdir)

    try:
        hdfs_prefix = hdfs_dir + "/" + sample_name
        hdfs_bam = hdfs_dir + "/" + bam_name

        hdfs_snps = hdfs_dir + "/" + inputs.dbsnp.split('://')[-1].split('/')[-1]

        if not inputs.run_local:
            download_data(job, master_ip, inputs, inputs.dbsnp, inputs.sample, hdfs_snps, hdfs_bam)
        else:
            copy_files([inputs.sample, inputs.dbsnp], inputs.local_dir)

        adam_input = hdfs_prefix + ".adam"
        adam_snps = hdfs_dir + "/snps.var.adam"
        adam_convert(job, master_ip, inputs, hdfs_bam, hdfs_snps, adam_input, adam_snps, spark_on_toil)

        adam_output = hdfs_prefix + ".processed.bam"
        adam_transform(job, master_ip, inputs, adam_input, adam_snps, hdfs_dir, adam_output, spark_on_toil)

        out_file = inputs.output_dir + "/" + sample_name + inputs.suffix + ".bam"

        if not inputs.run_local:
            upload_data(job, master_ip, inputs, adam_output, out_file, spark_on_toil)
        else:
            local_adam_output = "%s/%s.processed.bam" % (inputs.local_dir, sample_name)
            move_files([local_adam_output], inputs.output_dir)

        remove_file(master_ip, hdfs_subdir, spark_on_toil)
    except:
        remove_file(master_ip, hdfs_subdir, spark_on_toil)
        raise
495171
def get_ogr_driver(filepath):
    
    filename, file_extension = os.path.splitext(filepath)
    EXTENSION = file_extension[1:]

    ogr_driver_count = ogr.GetDriverCount()
    for idx in range(ogr_driver_count):
        driver = ogr.GetDriver(idx)
        driver_extension = driver.GetMetadataItem(str('DMD_EXTENSION')) or ''
        driver_extensions = driver.GetMetadataItem(str('DMD_EXTENSIONS')) or ''

        if EXTENSION == driver_extension or EXTENSION in driver_extensions:
            return driver

    else:
        msg = 'No driver found for the following file extension: {}'.format(
            EXTENSION)
        raise ValueError(msg)
765710
def download_go_basic_obo(obo="go-basic.obo", prt=sys.stdout, loading_bar=True):
    """"""
    if not os.path.isfile(obo):
        http = "http://purl.obolibrary.org/obo/go"
        if "slim" in obo:
            http = "http://www.geneontology.org/ontology/subsets"
            # http = 'http://current.geneontology.org/ontology/subsets'
        obo_remote = "{HTTP}/{OBO}".format(HTTP=http, OBO=os.path.basename(obo))
        dnld_file(obo_remote, obo, prt, loading_bar)
    else:
        if prt is not None:
            prt.write("  EXISTS: {FILE}\n".format(FILE=obo))
    return obo