// downloadChunk downloads the chunk from s3
3801
func (d *downloader) downloadChunk(chunk dlchunk) error {
	in := &s3.GetObjectInput{}
	awsutil.Copy(in, d.in)

	// Get the next byte range of data
	in.Range = aws.String(chunk.ByteRange())

	var n int64
	var err error
	for retry := 0; retry <= d.partBodyMaxRetries; retry++ {
		var resp *s3.GetObjectOutput
		resp, err = d.cfg.S3.GetObjectWithContext(d.ctx, in, d.cfg.RequestOptions...)
		if err != nil {
			return err
		}
		d.setTotalBytes(resp) // Set total if not yet set.

		n, err = io.Copy(&chunk, resp.Body)
		resp.Body.Close()
		if err == nil {
			break
		}

		chunk.cur = 0
		logMessage(d.cfg.S3, aws.LogDebugWithRequestRetries,
			fmt.Sprintf("DEBUG: object part body download interrupted %s, err, %v, retrying attempt %d",
				aws.StringValue(in.Key), err, retry))
	}

	d.incrWritten(n)

	return err
}
895874
func (c *Client) Download(name string) (io.ReadCloser, error) {
	out, err := c.Files.Download(&dropbox.DownloadInput{name})
	if err != nil {
		return nil, err
	}

	return out.Body, nil
}
565581
func (e *StreamAggExec) appendResult2Chunk(chk *chunk.Chunk) error {
	for i, aggFunc := range e.aggFuncs {
		err := aggFunc.AppendFinalResult2Chunk(e.ctx, e.partialResults[i], chk)
		if err != nil {
			return err
		}
		aggFunc.ResetPartialResult(e.partialResults[i])
	}
	if len(e.aggFuncs) == 0 {
		chk.SetNumVirtualRows(chk.NumRows() + 1)
	}
	return nil
}
128482
func chunk(s []interface{}, sz int) [][]interface{} {
	r := [][]interface{}{}
	j := len(s)
	for i := 0; i < j; i += sz {
		r = append(r, s[i:i+sz])
	}
	return r
}
1019780
func (c *Context) DownloadURL(url string) (filePath string, err error) {

	ext := filepath.Ext(url)
	out, err := ioutil.TempFile("", fmt.Sprintf("%d_%d", c.Bot().ID, c.Chat.ID))

	if err != nil {
		return "", err
	}

	out.Close()
	os.Rename(out.Name(), out.Name()+ext)

	out, err = os.OpenFile(out.Name()+ext, os.O_RDWR, 0666)
	if err != nil {
		return "", err
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		return "", err
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return "", errors.New("non 2xx resp status")
	}
	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return "", err
	}

	return out.Name(), nil
}
804479
func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
	host := handlers.GetSourceIP(r)

	ctx := newContext(r, w, "WebDownloadZip")
	defer logger.AuditLog(w, r, "WebDownloadZip", mustGetClaimsFromToken(r))

	var wg sync.WaitGroup
	objectAPI := web.ObjectAPI()
	if objectAPI == nil {
		writeWebErrorResponse(w, errServerNotInitialized)
		return
	}

	// Auth is done after reading the body to accommodate for anonymous requests
	// when bucket policy is enabled.
	var args DownloadZipArgs
	tenKB := 10 * 1024 // To limit r.Body to take care of misbehaving anonymous client.
	decodeErr := json.NewDecoder(io.LimitReader(r.Body, int64(tenKB))).Decode(&args)
	if decodeErr != nil {
		writeWebErrorResponse(w, decodeErr)
		return
	}

	token := r.URL.Query().Get("token")
	claims, owner, authErr := webTokenAuthenticate(token)
	if authErr != nil {
		if authErr == errNoAuthToken {
			for _, object := range args.Objects {
				// Check if anonymous (non-owner) has access to download objects.
				if !globalPolicySys.IsAllowed(policy.Args{
					Action:          policy.GetObjectAction,
					BucketName:      args.BucketName,
					ConditionValues: getConditionValues(r, "", ""),
					IsOwner:         false,
					ObjectName:      pathJoin(args.Prefix, object),
				}) {
					writeWebErrorResponse(w, errAuthentication)
					return
				}
			}
		} else {
			writeWebErrorResponse(w, authErr)
			return
		}
	}

	// For authenticated users apply IAM policy.
	if authErr == nil {
		for _, object := range args.Objects {
			if !globalIAMSys.IsAllowed(iampolicy.Args{
				AccountName:     claims.Subject,
				Action:          iampolicy.GetObjectAction,
				BucketName:      args.BucketName,
				ConditionValues: getConditionValues(r, "", claims.Subject),
				IsOwner:         owner,
				ObjectName:      pathJoin(args.Prefix, object),
			}) {
				writeWebErrorResponse(w, errAuthentication)
				return
			}
		}
	}

	// Check if bucket is a reserved bucket name or invalid.
	if isReservedOrInvalidBucket(args.BucketName, false) {
		writeWebErrorResponse(w, errInvalidBucketName)
		return
	}

	getObject := objectAPI.GetObject
	if web.CacheAPI() != nil {
		getObject = web.CacheAPI().GetObject
	}
	listObjects := objectAPI.ListObjects
	if web.CacheAPI() != nil {
		listObjects = web.CacheAPI().ListObjects
	}

	archive := zip.NewWriter(w)
	defer archive.Close()

	getObjectInfo := objectAPI.GetObjectInfo
	if web.CacheAPI() != nil {
		getObjectInfo = web.CacheAPI().GetObjectInfo
	}
	opts := ObjectOptions{}
	var length int64
	for _, object := range args.Objects {
		// Writes compressed object file to the response.
		zipit := func(objectName string) error {
			info, err := getObjectInfo(ctx, args.BucketName, objectName, opts)
			if err != nil {
				return err
			}
			length = info.Size
			if objectAPI.IsEncryptionSupported() {
				if _, err = DecryptObjectInfo(&info, r.Header); err != nil {
					writeWebErrorResponse(w, err)
					return err
				}
				if crypto.IsEncrypted(info.UserDefined) {
					length, _ = info.DecryptedSize()
				}
			}
			length = info.Size
			var actualSize int64
			if info.IsCompressed() {
				// Read the decompressed size from the meta.json.
				actualSize = info.GetActualSize()
				// Set the info.Size to the actualSize.
				info.Size = actualSize
			}
			header := &zip.FileHeader{
				Name:               strings.TrimPrefix(objectName, args.Prefix),
				Method:             zip.Deflate,
				UncompressedSize64: uint64(length),
				UncompressedSize:   uint32(length),
			}
			zipWriter, err := archive.CreateHeader(header)
			if err != nil {
				writeWebErrorResponse(w, errUnexpected)
				return err
			}
			var startOffset int64
			var writer io.Writer

			if info.IsCompressed() {
				// The decompress metrics are set.
				snappyStartOffset := 0
				snappyLength := actualSize

				// Open a pipe for compression
				// Where compressWriter is actually passed to the getObject
				decompressReader, compressWriter := io.Pipe()
				snappyReader := snappy.NewReader(decompressReader)

				// The limit is set to the actual size.
				responseWriter := ioutil.LimitedWriter(zipWriter, int64(snappyStartOffset), snappyLength)
				wg.Add(1) //For closures.
				go func() {
					defer wg.Done()
					// Finally, writes to the client.
					_, perr := io.Copy(responseWriter, snappyReader)

					// Close the compressWriter if the data is read already.
					// Closing the pipe, releases the writer passed to the getObject.
					compressWriter.CloseWithError(perr)
				}()
				writer = compressWriter
			} else {
				writer = zipWriter
			}
			if objectAPI.IsEncryptionSupported() && crypto.S3.IsEncrypted(info.UserDefined) {
				// Response writer should be limited early on for decryption upto required length,
				// additionally also skipping mod(offset)64KiB boundaries.
				writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
				writer, startOffset, length, err = DecryptBlocksRequest(writer, r,
					args.BucketName, objectName, startOffset, length, info, false)
				if err != nil {
					writeWebErrorResponse(w, err)
					return err
				}
			}
			httpWriter := ioutil.WriteOnClose(writer)
			if err = getObject(ctx, args.BucketName, objectName, startOffset, length, httpWriter, "", opts); err != nil {
				httpWriter.Close()
				if info.IsCompressed() {
					// Wait for decompression go-routine to retire.
					wg.Wait()
				}
				return err
			}
			if err = httpWriter.Close(); err != nil {
				if !httpWriter.HasWritten() { // write error response only if no data has been written to client yet
					writeWebErrorResponse(w, err)
					return err
				}
			}
			if info.IsCompressed() {
				// Wait for decompression go-routine to retire.
				wg.Wait()
			}

			// Notify object accessed via a GET request.
			sendEvent(eventArgs{
				EventName:    event.ObjectAccessedGet,
				BucketName:   args.BucketName,
				Object:       info,
				ReqParams:    extractReqParams(r),
				RespElements: extractRespElements(w),
				UserAgent:    r.UserAgent(),
				Host:         host,
			})

			return nil
		}

		if !hasSuffix(object, slashSeparator) {
			// If not a directory, compress the file and write it to response.
			err := zipit(pathJoin(args.Prefix, object))
			if err != nil {
				return
			}
			continue
		}

		// For directories, list the contents recursively and write the objects as compressed
		// date to the response writer.
		marker := ""
		for {
			lo, err := listObjects(context.Background(), args.BucketName, pathJoin(args.Prefix, object), marker, "", 1000)
			if err != nil {
				return
			}
			marker = lo.NextMarker
			for _, obj := range lo.Objects {
				err = zipit(obj.Name)
				if err != nil {
					return
				}
			}
			if !lo.IsTruncated {
				break
			}
		}
	}
}
1287485
func (bow *Browser) Download(o io.Writer) (int64, error) {
	buff := bytes.NewBuffer(bow.body)
	return io.Copy(o, buff)
}
264651
func (c *Client) DownloadFile(ctx context.Context, file string, u *url.URL, param *Download) error {
	var err error
	if param == nil {
		param = &DefaultDownload
	}

	rc, contentLength, err := c.Download(ctx, u, param)
	if err != nil {
		return err
	}

	return c.WriteFile(ctx, file, rc, contentLength, param.Progress, param.Writer)
}
1070764
func (r *Registry) DownloadLayer(ctx context.Context, repository string, digest digest.Digest) (io.ReadCloser, error) {
	url := r.url("/v2/%s/blobs/%s", repository, digest)
	r.Logf("registry.layer.download url=%s repository=%s digest=%s", url, repository, digest)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	resp, err := r.Client.Do(req.WithContext(ctx))
	if err != nil {
		return nil, err
	}

	return resp.Body, nil
}
332584
func (r *RepositoryLicense) GetDownloadURL() string {
	if r == nil || r.DownloadURL == nil {
		return ""
	}
	return *r.DownloadURL
}
408462
func (c *tufClient) downloadSnapshot() error {
	logrus.Debug("Loading snapshot...")
	role := data.CanonicalSnapshotRole
	consistentInfo := c.newBuilder.GetConsistentInfo(role)

	_, err := c.tryLoadCacheThenRemote(consistentInfo)
	return err
}
1085660
func (e *baseExecutor) newFirstChunk() *chunk.Chunk {
	return chunk.New(e.retTypes(), e.initCap, e.maxChunkSize)
}
113993
func compileChunk(context *funcContext, chunk []ast.Stmt) { // {{{
	for _, stmt := range chunk {
		compileStmt(context, stmt)
	}
}
1207339
func Download(pkgSrc PkgSrc, dstFilePath string, w io.Writer) (rErr error) {
	wantChecksum := pkgSrc.Checksum()
	if info, err := os.Stat(dstFilePath); err == nil {
		if info.IsDir() {
			return errors.Errorf("destination path %s already exists and is a directory", dstFilePath)
		}
		if wantChecksum != "" {
			// if file already exists at destination and checksum is known, verify checksum of existing file.
			// If it matches, use existing file.
			checksum, err := computeSHA256Checksum(dstFilePath)
			if err != nil {
				return errors.Wrapf(err, "failed to compute checksum of %s", dstFilePath)
			}
			if checksum == wantChecksum {
				return nil
			}
		}
	}

	// open reader from source
	r, size, err := pkgSrc.Reader()
	if err != nil {
		return err
	}
	defer func() {
		if err := r.Close(); err != nil && rErr == nil {
			rErr = errors.Wrapf(err, "failed to close reader for %s in defer", pkgSrc.Path())
		}
	}()

	// create new file for package (overwrite any existing file)
	dstFile, err := os.Create(dstFilePath)
	if err != nil {
		return errors.Wrapf(err, "failed to create file %s", dstFilePath)
	}
	defer func() {
		if err := dstFile.Close(); err != nil && rErr == nil {
			rErr = errors.Wrapf(err, "failed to close file %s in defer", dstFilePath)
		}
	}()

	h := sha256.New()
	mw := io.MultiWriter(h, dstFile)

	_, _ = fmt.Fprintf(w, "Getting package from %v...\n", pkgSrc.Path())
	if err := copyWithProgress(mw, r, size, w); err != nil {
		return errors.Wrapf(err, "failed to copy package %s to %s", pkgSrc.Path(), dstFilePath)
	}

	// verify checksum if provided
	if wantChecksum != "" {
		actualChecksum := hex.EncodeToString(h.Sum(nil))
		if wantChecksum != actualChecksum {
			return errors.Errorf("SHA-256 checksum of downloaded package did not match expected checksum: expected %s, was %s", wantChecksum, actualChecksum)
		}
	}
	return nil
}
596504
func (ef *File) FetchChunk(offset int64) (Chunk, error) {
	ef.Lock()
	defer ef.Unlock()
	c := NewChunk()
	GoToSeeker(ef.file, offset)
	c.Offset = offset
	c.Data = make([]byte, ChunkSize)
	if _, err := ef.file.Read(c.Data); err != nil {
		return c, err
	}
	reader := bytes.NewReader(c.Data)
	c.ParseChunkHeader(reader)
	// Go to after Header
	GoToSeeker(reader, int64(c.Header.SizeHeader))
	c.ParseStringTable(reader)
	if err := c.ParseTemplateTable(reader); err != nil {
		return c, err
	}
	if err := c.ParseEventOffsets(reader); err != nil {
		return c, err
	}
	return c, nil
}
529939
func (w *l2capWriter) Chunk() {
	if w.chunked {
		panic("l2capWriter: chunk called twice without committing")
	}
	w.chunked = true
	if w.chunk == nil {
		w.chunk = make([]byte, 0, w.mtu)
	}
}
1181923
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) {
	uri := c.Gateway + "/bzz-raw:/" + hash
	res, err := http.DefaultClient.Get(uri)
	if err != nil {
		return nil, false, err
	}
	if res.StatusCode != http.StatusOK {
		res.Body.Close()
		return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status)
	}
	isEncrypted := (res.Header.Get("X-Decrypted") == "true")
	return res.Body, isEncrypted, nil
}