Ensure we have programs needed to download/manipulate the data
500155
def requirements_check():
    
    required_programs = [
        ('samtools',
         'http://samtools.sourceforge.net/'),
        ('bedtools',
         'http://bedtools.readthedocs.org/en/latest/'),
        ('bigWigToBedGraph',
         'http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/'),
        ('bedGraphToBigWig',
         'http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/'),
    ]
    for req, url in required_programs:
        try:
            p = subprocess.Popen(
                [req], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
            stdout, stderr = p.communicate()
        except OSError:
            raise ValueError("Please install %s (%s)" % (req, url))
924222
function main(program) {
  program || (program = { args: [] });
  if(program.args.length === 0) {
    program.outputHelp();
    return process.exit(1);
  }

  return Promise
    .map(program.args, _.partial(exports.downloadPosting, program))
    .map(function(html) {
      return cheerio.load(html);
    })
    .map(exports.extractInfo)
    .then(_.partial(exports.outputInfo, program));
}
873592
def _download_and_clean_file(filename, url):
  """"""
  temp_file, _ = urllib.request.urlretrieve(url)
  with tf.gfile.Open(temp_file, 'r') as temp_eval_file:
    with tf.gfile.Open(filename, 'w') as eval_file:
      for line in temp_eval_file:
        line = line.strip()
        line = line.replace(', ', ',')
        if not line or ',' not in line:
          continue
        if line[-1] == '.':
          line = line[:-1]
        line += '\n'
        eval_file.write(line)
  tf.gfile.Remove(temp_file)
1372199
def main():
    """"""
    options = _parse_args()
    archive = download_setuptools(
        version=options.version,
        download_base=options.download_base,
        downloader_factory=options.downloader_factory,
    )
    return _install(archive, _build_install_args(options))
1211217
def maybe_download(url, work_directory):
    """"""
    filename = url.split("/")[-1]
    filepath = os.path.join(work_directory, filename)
    if not os.path.exists(filepath):
        logger.info("Downloading to {}...".format(filepath))
        download(url, work_directory)
    return filepath
658213
def check_datafile_present_and_download(path, backup_url=None):
    
    path = Path(path)
    if path.is_file(): return True
    if backup_url is None: return False
    logg.info('try downloading from url\n' + backup_url + '\n' +
              '... this may take a while but only happens once')
    if not path.parent.is_dir():
        logg.info('creating directory', str(path.parent) + '/', 'for saving data')
        path.parent.mkdir(parents=True)
    from urllib.request import urlretrieve
    urlretrieve(backup_url, str(path), reporthook=download_progress)
    logg.info('')
    return True
17462
def ensure_file(url, path, md5sum=None):
    

    if not os.path.isfile(path) or (md5sum and md5sum != file_md5(path)):
        download_file(url, path)
169588
def downloadFiles(self, prompt=True, extract=False):
		""""""
		#First, get the download urls
		data = self.data
		downloadUrls = self.getDownloadUrls()
		
		#Then, confirm the user wants to do this
		if prompt:
			confirm = raw_input("Download files [Y/N]? ")
			if confirm.lower() != 'y':
				self.printd("Operation aborted by user input")
				return
				
		#Now, if they still do, do all this stuff:
		counter = -1
		for datum in data:
			counter += 1
			try:
				download = downloadUrls[counter]
			except:
				pass
				
			# Download the file; fix our user agent
			self.printd("Downloading " + datum[0] + " from " + download)
			headers = { 'User-Agent' : 'calcpkg/2.0' }
			request = urllib2.Request(download, None, headers)
			fileData = urllib2.urlopen(request).read()
			
			# Now, process the downloaded file
			dowName = datum[0]
			# Use a helper function to remove /pub, /files
			dowName = util.removeRootFromName(dowName)
			dowName = dowName[1:]
			dowName = dowName.replace('/', '-')
			dowName = self.downloadDir + dowName
			try:
				downloaded = open(dowName, 'wb')
			except:
				os.remove(dowName)
			downloaded.write(fileData)
			downloaded.close()
			self.printd("Download complete! Wrote file " + dowName + "\n")

			#Extract them if told to do so
			if extract:
				extractType = ""
				if '.zip' in dowName:
					extractType = "zip"
				elif '.tar' in dowName:
					extractType = "tar"
					specType = ""
					if '.bz2' in dowName:
						specType = ":bz2"
					elif ".gz" in dowName:
						specType = ":gz"
				elif ".tgz" in dowName:
					extractType = "tar"
					specType = ":gz"

				if extractType != "":
					self.printd("Extracting file " + dowName + ", creating directory for extracted files")
					dirName, a, ending = dowName.partition('.')
					dirName = dirName + '-' + ending
					try:
						os.mkdir(dirName)
					except:
						pass
					if extractType == "zip":
						archive = zipfile.ZipFile(dowName, 'r')
					elif extractType == "tar":
						archive = tarfile.open(dowName, "r" + specType)
					else:
						self.printd("An unknown error has occured!")
						return
					archive.extractall(dirName)
					self.printd("All files in archive extracted to " + dirName)
					os.remove(dowName)
					self.printd("The archive file " + dowName + " has been deleted!\n")
402497
def download_dataset(tournament, unzip):
    """"""
    click.echo(napi.download_current_dataset(tournament=tournament, unzip=unzip))
210339
def authorize_download(dataset_name=None):
    """"""
    print(('Acquiring resource: ' + dataset_name))
    # TODO, check resource is in dictionary!
    print('')
    dr = data_resources[dataset_name]
    print('Details of data: ')
    print((dr['details']))
    print('')
    if dr['citation']:
        print('Please cite:')
        print((dr['citation']))
        print('')
    if dr['size']:
        print(('After downloading the data will take up ' + str(dr['size']) + ' bytes of space.'))
        print('')
    print(('Data will be stored in ' + os.path.join(data_path, dataset_name) + '.'))
    print('')
    if overide_manual_authorize:
        if dr['license']:
            print('You have agreed to the following license:')
            print((dr['license']))
            print('')
        return True
    else:
        if dr['license']:
            print('You must also agree to the following license:')
            print((dr['license']))
            print('')
        return prompt_user('Do you wish to proceed with the download? [yes/no]')
482483
def download(self):
        
        if not os.path.exists(self.data_path):
            # Create if it doesn't exist
            pathlib.Path(self.data_path).mkdir(parents=True, exist_ok=True)

        if not os.path.exists(self.text_path):
            http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())

            with open(self.text_path, 'wt') as fp:
                request = http.request('GET', self.url)
                content = request.data.decode('utf8')
                fp.write(content)

        if not os.path.exists(self.processed_path):
            with open(self.text_path, 'rt') as fp:
                content = fp.read()

            alphabet = sorted(set(content))

            index_to_character = {idx: c for idx, c in enumerate(alphabet, 1)}
            character_to_index = {c: idx for idx, c in enumerate(alphabet, 1)}

            content_encoded = np.array([character_to_index[c] for c in content], dtype=np.uint8)

            data_dict = {
                'alphabet': alphabet,
                'index_to_character': index_to_character,
                'character_to_index': character_to_index,
                'content_encoded': content_encoded
            }

            with open(self.processed_path, 'wb') as fp:
                torch.save(data_dict, fp)
        else:
            with open(self.processed_path, 'rb') as fp:
                data_dict = torch.load(fp)

        return data_dict
457677
def _download(file_in, file_out, blocksize, output=False):
    """"""
    with warnings.catch_warnings():
        warnings.filterwarnings('ignore')

        if output:
            try:
                from tqdm.autonotebook import tqdm
            except ImportError:
                logger.warn("Cache progress bar requires tqdm to be installed:"
                            " conda/pip install tqdm")
                output = False
        if output:
            try:
                file_size = file_in.fs.size(file_in.path)
                pbar_disabled = False
            except ValueError as err:
                logger.debug("File system error requesting size: {}".format(err))
                file_size = 0
                pbar_disabled = True
            for i in range(100):
                if i not in display:
                    display.add(i)
                    out = i
                    break
            pbar = tqdm(total=file_size // 2 ** 20, leave=False,
                        disable=pbar_disabled,
                        position=out, desc=os.path.basename(file_out.path),
                        mininterval=0.1,
                        bar_format=r'{n}/|/{l_bar}')

        logger.debug("Caching {}".format(file_in.path))
        with file_in as f1:
            with file_out as f2:
                data = True
                while data:
                    data = f1.read(blocksize)
                    f2.write(data)
                    if output:
                        pbar.update(len(data) // 2**20)
        if output:
            try:
                pbar.update(pbar.total - pbar.n)  # force to full
                pbar.close()
            except Exception as e:
                logger.debug('tqdm exception: %s' % e)
            finally:
                display.remove(out)
1281575
def download(self, file_to_be_downloaded, perform_download=True, download_to_path=None):
        
        response = self.get(
            '/path/data/', file_to_be_downloaded, raw=False)
        if not perform_download:
            # The caller can decide how to process the download of the data
            return response
        if not download_to_path:
            download_to_path = file_to_be_downloaded.split("/")[-1]
        # download uses shutil.copyfileobj to download, which copies
        # the data in chunks
        o = open(download_to_path, 'wb')
        return shutil.copyfileobj(response.raw, o)
30230
def download_data(job, master_ip, inputs, known_snps, bam, hdfs_snps, hdfs_bam):
    

    log.info("Downloading known sites file %s to %s.", known_snps, hdfs_snps)
    call_conductor(job, master_ip, known_snps, hdfs_snps, memory=inputs.memory)

    log.info("Downloading input BAM %s to %s.", bam, hdfs_bam)
    call_conductor(job, master_ip, bam, hdfs_bam, memory=inputs.memory)
686769
def do_download_datafiles(self, _):
        '''  '''
        contents = {"trialdata": lambda p: p.get_trial_data(), "eventdata": \
                    lambda p: p.get_event_data(), "questiondata": lambda p: \
                    p.get_question_data()}
        query = Participant.query.all()
        for k in contents:
            ret = "".join([contents[k](p) for p in query])
            temp_file = open(k + '.csv', 'w')
            temp_file.write(ret)
            temp_file.close()
916385
def call(self):
        
        data = self._downloader.download()

        # Only try to pass errors arg if supported
        if sys.version >= "2.7":
            data = data.decode("utf-8", errors="ignore")
        else:
            data = data.decode("utf-8")

        self.update(json.loads(data))
        self._fetched = True
125740
def _do_download(version, download_base, to_dir, download_delay):
    """"""
    py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys)
    tp = 'setuptools-{version}-{py_desig}.egg'
    egg = os.path.join(to_dir, tp.format(**locals()))
    if not os.path.exists(egg):
        archive = download_setuptools(version, download_base,
            to_dir, download_delay)
        _build_egg(egg, archive, to_dir)
    sys.path.insert(0, egg)

    # Remove previously-imported pkg_resources if present (see
    # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details).
    if 'pkg_resources' in sys.modules:
        _unload_pkg_resources()

    import setuptools
    setuptools.bootstrap_install_from = egg