Create a pipeline to process a file after downloading.
        (Optionally decrypt, then decompress)
358092
def get_download_pipeline(in_fd, out_fd, gpg=False, lzop=True):
    
    commands = []
    if gpg:
        commands.append(GPGDecryptionFilter())
    if lzop:
        commands.append(LZODecompressionFilter())
    return Pipeline(commands, in_fd, out_fd)
702380
def _decompress(self, fp):
        
        decompressor = zlib.decompressobj()
        if self.stream:
            return Proxy(decompressor.decompress, fp)
        else:
            out = io.BytesIO(decompressor.decompress(fp.read()))
            out.write(decompressor.flush())
            out.seek(0)
            return out
651947
def stream_download(self, chunk_size: Optional[int] = None, callback: Optional[Callable] = None) -> AsyncIterator[bytes]:
        
        pass
1122965
function () {
            if (this._isProcessing) {
                return;
            }
            this.$assert(33, this._logicalPaths.length > 0);
            this._isProcessing = true;
            (require("./IO")).asyncRequest({
                sender : {
                    classpath : this.$classpath,
                    logicalPaths : this._logicalPaths
                },
                url : require("./DownloadMgr").getURLWithTimestamp(this._url), // add a timestamp to the URL if
                // required
                callback : {
                    fn : this._onFileReceive,
                    onerror : this._onFileReceive,
                    scope : this
                },
                expectedResponseType : "text"
            });
        }
811429
def _handle_download_result(self, resource, tmp_dir_path, sha256, dl_size):
    """"""
    fnames = tf.io.gfile.listdir(tmp_dir_path)
    if len(fnames) > 1:
      raise AssertionError('More than one file in %s.' % tmp_dir_path)
    original_fname = fnames[0]
    tmp_path = os.path.join(tmp_dir_path, original_fname)
    self._recorded_sizes_checksums[resource.url] = (dl_size, sha256)
    if self._register_checksums:
      self._record_sizes_checksums()
    elif (dl_size, sha256) != self._sizes_checksums.get(resource.url, None):
      raise NonMatchingChecksumError(resource.url, tmp_path)
    download_path = self._get_final_dl_path(resource.url, sha256)
    resource_lib.write_info_file(resource, download_path, self._dataset_name,
                                 original_fname)
    # Unconditionally overwrite because either file doesn't exist or
    # FORCE_DOWNLOAD=true
    tf.io.gfile.rename(tmp_path, download_path, overwrite=True)
    tf.io.gfile.rmtree(tmp_dir_path)
    return download_path
261504
def download_from_url(source, destination, progress=False, uncompress=False):
    """"""
    # Modules #
    from tqdm import tqdm
    import requests
    from autopaths.file_path import FilePath
    # Check destination exists #
    destination = FilePath(destination)
    destination.directory.create_if_not_exists()
    # Over HTTP #
    response = requests.get(source, stream=True)
    total_size = int(response.headers.get('content-length'))
    block_size = int(total_size/1024)
    # Do it #
    with open(destination, "wb") as handle:
        if progress:
            for data in tqdm(response.iter_content(chunk_size=block_size), total=1024): handle.write(data)
        else:
            for data in response.iter_content(chunk_size=block_size): handle.write(data)
    # Uncompress #
    if uncompress:
        with open(destination) as f: header = f.read(4)
        if header == "PK\x03\x04": unzip(destination, inplace=True)
        # Add other compression formats here
    # Return #
    return destination
596881
def _download_and_decompress_if_necessary(
        full_path,
        download_url,
        timeout=None,
        use_wget_if_available=False):
    
    logger.info("Downloading %s to %s", download_url, full_path)
    filename = os.path.split(full_path)[1]
    base_name, ext = os.path.splitext(filename)
    tmp_path = _download_to_temp_file(
        download_url=download_url,
        timeout=timeout,
        base_name=base_name,
        ext=ext,
        use_wget_if_available=use_wget_if_available)

    if download_url.endswith("zip") and not filename.endswith("zip"):
        logger.info("Decompressing zip into %s...", filename)
        with zipfile.ZipFile(tmp_path) as z:
            names = z.namelist()
            assert len(names) > 0, "Empty zip archive"
            if filename in names:
                chosen_filename = filename
            else:
                # If zip archive contains multiple files, choose the biggest.
                biggest_size = 0
                chosen_filename = names[0]
                for info in z.infolist():
                    if info.file_size > biggest_size:
                        chosen_filename = info.filename
                        biggest_size = info.file_size
            extract_path = z.extract(chosen_filename)
        move(extract_path, full_path)
        os.remove(tmp_path)
    elif download_url.endswith("gz") and not filename.endswith("gz"):
        logger.info("Decompressing gzip into %s...", filename)
        with gzip.GzipFile(tmp_path) as src:
            contents = src.read()
        os.remove(tmp_path)
        with open(full_path, 'wb') as dst:
            dst.write(contents)
    elif download_url.endswith(("html", "htm")) and full_path.endswith(".csv"):
        logger.info("Extracting HTML table into CSV %s...", filename)
        df = pd.read_html(tmp_path, header=0)[0]
        df.to_csv(full_path, sep=',', index=False, encoding='utf-8')
    else:
        move(tmp_path, full_path)
568535
def maybe_download_and_extract():
  """"""
  dest_directory = "/tmp/cifar"
  if not os.path.exists(dest_directory):
    os.makedirs(dest_directory)
  filename = DATA_URL.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):
    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename,
          float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()
    filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)
    print()
    statinfo = os.stat(filepath)
    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
    tarfile.open(filepath, 'r:gz').extractall(dest_directory)
1253990
def maybe_download_and_extract(self):
        """"""
        if not os.path.exists(self.cifar10_dir):

            if not os.path.exists(self.data_dir):
                os.makedirs(self.data_dir)

            def _progress(count, block_size, total_size):
                status_msg = '\r>> Downloading {} {:>3}%   '
                sys.stdout.write(status_msg.format(self.cifar10_tarball, float(count * block_size) / total_size * 100.0))
                sys.stdout.flush()

            file_path, _ = urlretrieve(CIFAR10_URL, self.cifar10_tarball, _progress)

            stat_info = os.stat(file_path)
            print('\nSuccessfully downloaded', file_path, stat_info.st_size, 'bytes.\n')

            tarfile.open(file_path, 'r:gz').extractall(self.data_dir)
715281
def _decompress_data(self, data):
        ''''''
        if self._decompressor:
            try:
                return self._decompressor.decompress(data)
            except zlib.error as error:
                raise ProtocolError(
                    'zlib error: {0}.'.format(error)
                ) from error
        else:
            return data
202660
def stream_download(cookie, tokens, path):
    '''
    '''
    url = ''.join([
        const.PCS_URL_D,
        'file?method=download',
        '&path=', encoder.encode_uri_component(path),
        '&app_id=250528',
    ])
    req = net.urlopen_without_redirect(url, headers=
            {'Cookie': cookie.header_output()})
    if req:
        return req
    else:
        return None
734199
def download(self, to_path=None, name=None, chunk_size='auto',
                 convert_to_pdf=False):
        
        return super().download(to_path=to_path, name=name,
                                chunk_size=chunk_size,
                                convert_to_pdf=convert_to_pdf)
1300591
def postprocess_key_for_export(self, key, client_random, server_random,
                                   con_end, read_or_write, req_len):
        
        s = con_end + read_or_write
        s = (s == "clientwrite" or s == "serverread")

        if self.tls_version < 0x0300:
            return None
        elif self.tls_version == 0x0300:
            if s:
                tbh = key + client_random + server_random
            else:
                tbh = key + server_random + client_random
            export_key = _tls_hash_algs["MD5"]().digest(tbh)[:req_len]
        else:
            if s:
                tag = b"client write key"
            else:
                tag = b"server write key"
            export_key = self.prf(key,
                                  tag,
                                  client_random + server_random,
                                  req_len)
        return export_key
179358
def download(url, options = {})
      options = @options.merge(options)

      max_size            = options.delete(:max_size)
      max_redirects       = options.delete(:max_redirects)
      progress_proc       = options.delete(:progress_proc)
      content_length_proc = options.delete(:content_length_proc)
      destination         = options.delete(:destination)
      headers             = options.delete(:headers) || {}

      # Use open-uri's :content_lenth_proc or :progress_proc to raise an
      # exception early if the file is too large.
      #
      # Also disable following redirects, as we'll provide our own
      # implementation that has the ability to limit the number of redirects.
      open_uri_options = {
        content_length_proc: proc { |size|
          if size && max_size && size > max_size
            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
          end
          content_length_proc.call(size) if content_length_proc
        },
        progress_proc: proc { |current_size|
          if max_size && current_size > max_size
            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
          end
          progress_proc.call(current_size) if progress_proc
        },
        redirect: false,
      }

      # Handle basic authentication in the :proxy option.
      if options[:proxy]
        proxy    = URI(options.delete(:proxy))
        user     = proxy.user
        password = proxy.password

        if user || password
          proxy.user     = nil
          proxy.password = nil

          open_uri_options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
        else
          open_uri_options[:proxy] = proxy.to_s
        end
      end

      open_uri_options.merge!(options)
      open_uri_options.merge!(headers)

      uri = ensure_uri(addressable_normalize(url))

      # Handle basic authentication in the remote URL.
      if uri.user || uri.password
        open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
        uri.user = nil
        uri.password = nil
      end

      open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)

      # Handle the fact that open-uri returns StringIOs for small files.
      tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
      OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
      tempfile.extend Down::NetHttp::DownloadedFile

      download_result(tempfile, destination)
    end
1284483
def _download_if_necessary(self, url, download_if_missing, overwrite):
        
        cached_path = self.cached_path(url)
        missing = not exists(cached_path)
        if (missing or overwrite) and download_if_missing:
            logger.info("Fetching %s from URL %s", cached_path, url)
            datacache.download._download_and_decompress_if_necessary(
                full_path=cached_path,
                download_url=url,
                timeout=3600)
        elif missing:
            raise MissingRemoteFile(url)
        return cached_path
1017038
def decompressBWTPoolProcess(tup):
    '''
    
    '''
    (inputDir, outputDir, startIndex, endIndex) = tup
    
    if startIndex == endIndex:
        return True
    
    #load the thing we'll be extracting from
    msbwt = MultiStringBWT.CompressedMSBWT()
    msbwt.loadMsbwt(inputDir, None)
    
    #open our output
    outputBwt = np.load(outputDir+'/msbwt.npy', 'r+')
    outputBwt[startIndex:endIndex] = msbwt.getBWTRange(startIndex, endIndex)   
    
    return True
262381
def download(url, file_handle, chunk_size=1024):
    
    r = requests.get(url, stream=True)
    total_length = r.headers.get('content-length')
    if total_length is None:
        maxval = UnknownLength
    else:
        maxval = int(total_length)
    name = file_handle.name
    with progress_bar(name=name, maxval=maxval) as bar:
        for i, chunk in enumerate(r.iter_content(chunk_size)):
            if total_length:
                bar.update(i * chunk_size)
            file_handle.write(chunk)