// getTotalMultipartSize - calculate total uploaded size for the a given multipart object.
346857
func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) (size int64, err error) {
	// Iterate over all parts and aggregate the size.
	partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID)
	if err != nil {
		return 0, err
	}
	for _, partInfo := range partsInfo {
		size += partInfo.Size
	}
	return size, nil
}
45146
func (e *EtcdStore) GetMultiMap(prefix string) (map[string]io.Reader, error) {
	kv := make(map[string]io.Reader)
	recursive := true
	res, err := get(e, prefix, recursive)
	if err != nil {
		return kv, err
	}
	extractKeyValues(res.Node, kv)
	return kv, nil
}
158884
func (c *Consumer) PostMultipart(url, multipartName string, multipartData io.ReadCloser, userParams map[string]string, token *AccessToken) (resp *http.Response, err error) {
	return c.makeAuthorizedRequestReader("POST", url, LOC_MULTIPART, 0, multipartName, multipartData, userParams, token)
}
813959
func (s *Bucket) AbortMultipartUpload(objectKey string, input *AbortMultipartUploadInput) (*AbortMultipartUploadOutput, error) {
	r, x, err := s.AbortMultipartUploadRequest(objectKey, input)

	if err != nil {
		return x, err
	}

	err = r.Send()
	if err != nil {
		return nil, err
	}

	requestID := r.HTTPResponse.Header.Get(http.CanonicalHeaderKey("X-QS-Request-ID"))
	x.RequestID = &requestID

	return x, err
}
1004497
func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
	return lmi, NotImplemented{}
}
316549
func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
	metadataObject := getAzureMetadataObjectName(object, uploadID)
	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
		return objInfo, err
	}

	if err = checkAzureUploadID(ctx, uploadID); err != nil {
		return objInfo, err
	}

	var metadataReader io.Reader
	blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
	if metadataReader, err = blob.Get(nil); err != nil {
		return objInfo, azureToObjectError(err, bucket, metadataObject)
	}

	var metadata azureMultipartMetadata
	if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil {
		logger.LogIf(ctx, err)
		return objInfo, azureToObjectError(err, bucket, metadataObject)
	}

	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)

	var allBlocks []storage.Block
	for i, part := range uploadedParts {
		var partMetadataReader io.Reader
		var partMetadata partMetadataV1
		partMetadataObject := getAzureMetadataPartName(object, uploadID, part.PartNumber)
		pblob := a.client.GetContainerReference(bucket).GetBlobReference(partMetadataObject)
		if partMetadataReader, err = pblob.Get(nil); err != nil {
			return objInfo, azureToObjectError(err, bucket, partMetadataObject)
		}

		if err = json.NewDecoder(partMetadataReader).Decode(&partMetadata); err != nil {
			logger.LogIf(ctx, err)
			return objInfo, azureToObjectError(err, bucket, partMetadataObject)
		}

		if partMetadata.ETag != part.ETag {
			return objInfo, minio.InvalidPart{}
		}
		for _, blockID := range partMetadata.BlockIDs {
			allBlocks = append(allBlocks, storage.Block{ID: blockID, Status: storage.BlockStatusUncommitted})
		}
		if i < (len(uploadedParts)-1) && partMetadata.Size < azureS3MinPartSize {
			return objInfo, minio.PartTooSmall{
				PartNumber: uploadedParts[i].PartNumber,
				PartSize:   partMetadata.Size,
				PartETag:   uploadedParts[i].ETag,
			}
		}
	}

	err = objBlob.PutBlockList(allBlocks, nil)
	if err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}
	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata.Metadata)
	if err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}
	objBlob.Metadata["md5sum"], err = cmd.ComputeCompleteMultipartMD5(uploadedParts)
	if err != nil {
		return objInfo, err
	}
	err = objBlob.SetProperties(nil)
	if err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}
	err = objBlob.SetMetadata(nil)
	if err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}
	var partNumberMarker int
	for {
		lpi, err := a.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxPartsCount, minio.ObjectOptions{})
		if err != nil {
			break
		}
		for _, part := range lpi.Parts {
			pblob := a.client.GetContainerReference(bucket).GetBlobReference(
				getAzureMetadataPartName(object, uploadID, part.PartNumber))
			pblob.Delete(nil)
		}
		partNumberMarker = lpi.NextPartNumberMarker
		if !lpi.IsTruncated {
			break
		}
	}

	blob = a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
	derr := blob.Delete(nil)
	logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
	logger.LogIf(ctx, derr)

	return a.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
}
338432
func (r *Request) SetMultipartField(param, fileName, contentType string, reader io.Reader) *Request {
	r.isMultiPart = true
	r.multipartFields = append(r.multipartFields, &MultipartField{
		Param:       param,
		FileName:    fileName,
		ContentType: contentType,
		Reader:      reader,
	})
	return r
}
888613
func NewMultipartBuilder() MultipartBuilder {
	builder := MultipartBuilder{}
	var b bytes.Buffer
	builder.Buffer = &b
	builder.Writer = multipart.NewWriter(&b)
	return builder
}
543777
func (f *FS) uploadMultiPart(o *object, r io.Reader) error {

	//chunkSize, err := calcBlockSize(size)
	// if err != nil {
	// 	return err
	// }
	var buf = make([]byte, initialChunkSize)

	var blocks []az.Block
	var rawID uint64

	blob := f.client.GetContainerReference(f.bucket).GetBlobReference(o.name)

	// TODO: performance improvement to mange uploads in separate
	// go-routine than the reader
	for {
		n, err := r.Read(buf)
		if err != nil {
			if err == io.EOF {
				break
			}
			gou.Warnf("unknown err=%v", err)
			return err
		}

		blockID := makeBlockID(rawID)
		chunk := buf[:n]

		if err := blob.PutBlock(blockID, chunk, nil); err != nil {
			return err
		}

		blocks = append(blocks, az.Block{
			ID:     blockID,
			Status: az.BlockStatusLatest,
		})
		rawID++
	}

	err := blob.PutBlockList(blocks, nil)
	if err != nil {
		gou.Warnf("could not put block list %v", err)
		return err
	}

	err = blob.GetProperties(nil)
	if err != nil {
		gou.Warnf("could not load blog properties %v", err)
		return err
	}

	blob.Metadata = o.metadata

	err = blob.SetMetadata(nil)
	if err != nil {
		gou.Warnf("can't set metadata err=%v", err)
		return err
	}
	return nil
}
1253868
func (i *Import) GetLargeFilesSize() int {
	if i == nil || i.LargeFilesSize == nil {
		return 0
	}
	return *i.LargeFilesSize
}
1003727
func IsMultiPart(metadata map[string]string) bool {
	if _, ok := metadata[SSEMultipart]; ok {
		return true
	}
	return false
}
379171
func (d *Dataset) GetTotalSize() int64 {
	if d.Meta.TotalSize >= 0 {
		return d.Meta.TotalSize
	}
	var currentDatasetTotalSize int64
	for _, ds := range d.Step.InputDatasets {
		currentDatasetTotalSize += ds.GetTotalSize()
	}
	d.Meta.TotalSize = currentDatasetTotalSize
	return currentDatasetTotalSize
}
1340769
func getTotalMem() int64 {
	pagesize := C.sysconf(C._SC_PAGESIZE)
	npages := C.sysconf(C._SC_PHYS_PAGES)
	return int64(pagesize * npages)
}
247923
func (d *Dataset) GetPartitionSize() int64 {
	return d.GetTotalSize() / int64(len(d.Shards))
}
884660
func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, uploadID string, opts ObjectOptions) (string, error) {
	if cfs.diskUsageHigh() {
		select {
		case cfs.purgeChan <- struct{}{}:
		default:
		}
		return "", errDiskFull
	}

	if _, err := cfs.GetBucketInfo(ctx, bucket); err != nil {
		pErr := cfs.MakeBucketWithLocation(ctx, bucket, "")
		if pErr != nil {
			return "", pErr
		}
	}
	fs := cfs.FSObjects
	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
		return "", toObjectErr(err, bucket)
	}

	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
		return "", toObjectErr(err, bucket)
	}

	uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)

	err := mkdirAll(uploadIDDir, 0755)
	if err != nil {
		logger.LogIf(ctx, err)
		return "", err
	}

	// Initialize fs.json values.
	fsMeta := newFSMetaV1()
	fsMeta.Meta = opts.UserDefined

	fsMetaBytes, err := json.Marshal(fsMeta)
	if err != nil {
		logger.LogIf(ctx, err)
		return "", err
	}

	if err = ioutil.WriteFile(pathJoin(uploadIDDir, fs.metaJSONFile), fsMetaBytes, 0644); err != nil {
		logger.LogIf(ctx, err)
		return "", err
	}
	return uploadID, nil
}
1121589
func metadataTotalLength(types []byte) int {
	sum := 0
	for _, t := range types {
		sum += metadataLength(t)
	}
	return sum
}
722604
func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error {
	boundary := randomBoundary()
	hdr := http.Header{}
	hdr.Set("Content-Type", "multipart/form-data; boundary="+boundary)
	hdr.Set("User-Agent", r.collector.UserAgent)
	return r.collector.scrape(r.AbsoluteURL(URL), "POST", r.Depth+1, createMultipartReader(boundary, requestData), r.Ctx, hdr, true)
}