// CopyObjectPart creates a part in a multipart upload by copying
// existing object or a part of it.
358758
func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
		return minio.PartInfo{}, minio.PreConditionFailed{}
	}
	srcInfo.UserDefined = map[string]string{
		"x-amz-copy-source-if-match": srcInfo.ETag,
	}
	header := make(http.Header)
	if srcOpts.ServerSideEncryption != nil {
		encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
	}

	if dstOpts.ServerSideEncryption != nil {
		dstOpts.ServerSideEncryption.Marshal(header)
	}
	for k, v := range header {
		srcInfo.UserDefined[k] = v[0]
	}

	completePart, err := l.Client.CopyObjectPart(srcBucket, srcObject, destBucket, destObject,
		uploadID, partID, startOffset, length, srcInfo.UserDefined)
	if err != nil {
		return p, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
	}
	p.PartNumber = completePart.PartNumber
	p.ETag = completePart.ETag
	return p, nil
}
936431
func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
	bkt, err := l.Bucket(ctx, bucket)
	if err != nil {
		return oi, err
	}
	hashes := make(map[int]string)
	for i, uploadedPart := range uploadedParts {
		// B2 requires contigous part numbers starting with 1, they do not support
		// hand picking part numbers, we return an S3 compatible error instead.
		if i+1 != uploadedPart.PartNumber {
			logger.LogIf(ctx, minio.InvalidPart{})
			return oi, b2ToObjectError(minio.InvalidPart{}, bucket, object, uploadID)
		}

		// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
		hashes[uploadedPart.PartNumber] = strings.TrimSuffix(uploadedPart.ETag, "-1")
	}

	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
		logger.LogIf(ctx, err)
		return oi, b2ToObjectError(err, bucket, object, uploadID)
	}

	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
}
377126
func (m *multipartBody) NextPart() (*Entity, error) {
	if m.i >= len(m.parts) {
		return nil, io.EOF
	}

	part := m.parts[m.i]
	m.i++
	return part, nil
}
1249362
func (a *S3Store) uploadPart(ctx context.Context, task *UploadTask, b job, previous *AttachmentInfo, previousParts map[int]s3.Part, multi s3.MultiInt, retCh chan s3.Part) (err error) {
	defer a.Trace(ctx, func() error { return err }, fmt.Sprintf("uploadPart(%d)", b.index))()

	// check to see if this part has already been uploaded.
	// for job `b` to be here, it has already passed local stash verification.
	if previous != nil {
		// check s3 previousParts for this block
		p, ok := previousParts[b.index]
		if ok && int(p.Size) == len(b.block) && p.ETag == b.etag() {
			a.Debug(ctx, "uploadPart: part %d already uploaded to s3", b.index)

			// part already uploaded, so put it in the retCh unless the context
			// has been canceled
			select {
			case retCh <- p:
			case <-ctx.Done():
				return ctx.Err()
			}

			// nothing else to do
			return nil
		}

		if p.Size > 0 {
			// only abort if the part size from s3 is > 0.
			a.Debug(ctx, "uploadPart: part %d s3 mismatch:  size %d != expected %d or etag %s != expected %s",
				b.index, p.Size, len(b.block), p.ETag, b.etag())
			return ErrAbortOnPartMismatch
		}

		// this part doesn't exist on s3, so it needs to be uploaded
		a.Debug(ctx, "uploadPart: part %d not uploaded to s3 by previous upload attempt", b.index)
	}

	// stash part info locally before attempting S3 put
	// doing this before attempting the S3 put is important
	// for security concerns.
	if err := a.stash.RecordPart(task.stashKey(), b.index, b.hash); err != nil {
		a.Debug(ctx, "uploadPart: StashRecordPart error: %s", err)
	}

	part, putErr := multi.PutPart(ctx, b.index, bytes.NewReader(b.block))
	if putErr != nil {
		return NewErrorWrapper(fmt.Sprintf("failed to put part %d", b.index), putErr)
	}

	// put the successfully uploaded part information in the retCh
	// unless the context has been canceled.
	select {
	case retCh <- part:
	case <-ctx.Done():
		a.Debug(ctx, "uploadPart: upload part %d, context canceled", b.index)
		return ctx.Err()
	}

	return nil
}
748861
func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) {
	lupr, err := ossListObjectParts(l.Client, bucket, object, uploadID, partNumberMarker, maxParts)
	if err != nil {
		logger.LogIf(ctx, err)
		return lpi, ossToObjectError(err, bucket, object, uploadID)
	}

	return fromOSSClientListPartsInfo(lupr, partNumberMarker), nil
}
570198
func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) {
	bkt, err := l.Bucket(ctx, bucket)
	if err != nil {
		return lpi, err
	}
	lpi = minio.ListPartsInfo{
		Bucket:           bucket,
		Object:           object,
		UploadID:         uploadID,
		MaxParts:         maxParts,
		PartNumberMarker: partNumberMarker,
	}
	// startPartNumber must be in the range 1 - 10000 for B2.
	partNumberMarker++
	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
	if err != nil {
		logger.LogIf(ctx, err)
		return lpi, b2ToObjectError(err, bucket, object, uploadID)
	}
	if next != 0 {
		lpi.IsTruncated = true
		lpi.NextPartNumberMarker = next
	}
	for _, part := range partsList {
		lpi.Parts = append(lpi.Parts, minio.PartInfo{
			PartNumber: part.Number,
			ETag:       minio.ToS3ETag(part.SHA1),
			Size:       part.Size,
		})
	}
	return lpi, nil
}
381725
func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string,
	metadata map[string]string) (ObjectInfo, error) {

	// Build headers.
	headers := make(http.Header)

	// Set all the metadata headers.
	for k, v := range metadata {
		headers.Set(k, v)
	}

	// Set the source header
	headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject))

	// Send upload-part-copy request
	resp, err := c.executeMethod(ctx, "PUT", requestMetadata{
		bucketName:   destBucket,
		objectName:   destObject,
		customHeader: headers,
	})
	defer closeResponse(resp)
	if err != nil {
		return ObjectInfo{}, err
	}

	// Check if we got an error response.
	if resp.StatusCode != http.StatusOK {
		return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject)
	}

	cpObjRes := copyObjectResult{}
	err = xmlDecoder(resp.Body, &cpObjRes)
	if err != nil {
		return ObjectInfo{}, err
	}

	objInfo := ObjectInfo{
		Key:          destObject,
		ETag:         strings.Trim(cpObjRes.ETag, "\""),
		LastModified: cpObjRes.LastModified,
	}
	return objInfo, nil
}
837385
func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
		return minio.ObjectInfo{}, minio.PreConditionFailed{}
	}
	// Set this header such that following CopyObject() always sets the right metadata on the destination.
	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
	// So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API.
	srcInfo.UserDefined["x-amz-metadata-directive"] = "REPLACE"
	srcInfo.UserDefined["x-amz-copy-source-if-match"] = srcInfo.ETag
	header := make(http.Header)
	if srcOpts.ServerSideEncryption != nil {
		encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
	}

	if dstOpts.ServerSideEncryption != nil {
		dstOpts.ServerSideEncryption.Marshal(header)
	}
	for k, v := range header {
		srcInfo.UserDefined[k] = v[0]
	}

	if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil {
		return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
	}
	return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts)
}
714232
func copyComponent(c Component) Component {
	if c == nil {
		panic("vecty: internal error (cannot copy nil Component)")
	}

	// If the Component implements the Copier interface, then use that to
	// perform the copy.
	if copier, ok := c.(Copier); ok {
		cpy := copier.Copy()
		if cpy == c {
			panic("vecty: Component.Copy illegally returned an identical *MyComponent pointer")
		}
		return cpy
	}

	// Component does not implement the Copier interface, so perform a shallow
	// copy.
	v := reflect.ValueOf(c)
	if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
		panic("vecty: Component must be pointer to struct, found " + reflect.TypeOf(c).String())
	}
	cpy := reflect.New(v.Elem().Type())
	cpy.Elem().Set(v.Elem())
	return cpy.Interface().(Component)
}
443955
func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
	data := r.Reader
	if data.Size() < azureBlockSize/10 {
		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
		if err != nil {
			return objInfo, azureToObjectError(err, bucket, object)
		}
		if err = blob.CreateBlockBlobFromReader(data, nil); err != nil {
			return objInfo, azureToObjectError(err, bucket, object)
		}
		return a.GetObjectInfo(ctx, bucket, object, opts)
	}

	blockIDs := make(map[string]string)

	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
	subPartSize, subPartNumber := int64(azureBlockSize), 1
	for remainingSize := data.Size(); remainingSize >= 0; remainingSize -= subPartSize {
		// Allow to create zero sized part.
		if remainingSize == 0 && subPartNumber > 1 {
			break
		}

		if remainingSize < subPartSize {
			subPartSize = remainingSize
		}

		id := base64.StdEncoding.EncodeToString([]byte(minio.MustGetUUID()))
		blockIDs[id] = ""
		if err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil); err != nil {
			return objInfo, azureToObjectError(err, bucket, object)
		}
		subPartNumber++
	}

	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
	if err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}
	getBlocks := func(blocksMap map[string]string) (blocks []storage.Block, size int64, aerr error) {
		for _, part := range resp.UncommittedBlocks {
			if _, ok := blocksMap[part.Name]; ok {
				blocks = append(blocks, storage.Block{
					ID:     part.Name,
					Status: storage.BlockStatusUncommitted,
				})

				size += part.Size
			}
		}

		if len(blocks) == 0 {
			return nil, 0, minio.InvalidPart{}
		}

		return blocks, size, nil
	}

	var blocks []storage.Block
	blocks, _, err = getBlocks(blockIDs)
	if err != nil {
		logger.LogIf(ctx, err)
		return objInfo, err
	}

	if err = objBlob.PutBlockList(blocks, nil); err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}

	if len(opts.UserDefined) == 0 {
		opts.UserDefined = map[string]string{}
	}

	// Save md5sum for future processing on the object.
	opts.UserDefined["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
	if err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}
	if err = objBlob.SetProperties(nil); err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}
	if err = objBlob.SetMetadata(nil); err != nil {
		return objInfo, azureToObjectError(err, bucket, object)
	}

	return a.GetObjectInfo(ctx, bucket, object, opts)
}
615242
func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
	// Hold read locks on source object only if we are
	// going to read data from source object.
	objectSRLock := xl.nsMutex.NewNSLock(srcBucket, srcObject)
	if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
		return pi, err
	}
	defer objectSRLock.RUnlock()

	if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, xl); err != nil {
		return pi, err
	}

	partInfo, err := xl.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
	if err != nil {
		return pi, toObjectErr(err, dstBucket, dstObject)
	}

	// Success.
	return partInfo, nil
}
927980
func (b *Bucket) CopyLargeFileInParallel(sourcePath string, destPath string, contentType string, perm ACL, options Options, maxConcurrency int) error {

	if maxConcurrency < 1 {
		maxConcurrency = 1
	}

	currentLength, err := b.GetContentLength(sourcePath)

	log.Printf("Parallel Copy large file[size: %d] from %s to %s\n", currentLength, sourcePath, destPath)

	if err != nil {
		return err
	}

	if currentLength < maxCopytSize {
		_, err := b.PutCopy(destPath, perm,
			CopyOptions{},
			b.Path(sourcePath))
		return err
	}

	multi, err := b.InitMulti(destPath, contentType, perm, options)
	if err != nil {
		return err
	}

	numParts := (currentLength + defaultChunkSize - 1) / defaultChunkSize
	completedParts := make([]Part, numParts)

	errChan := make(chan error, numParts)
	limiter := make(chan struct{}, maxConcurrency)

	var start int64 = 0
	var to int64 = 0
	var partNumber = 0
	sourcePathForCopy := b.Path(sourcePath)

	for start = 0; start < currentLength; start = to {
		to = start + defaultChunkSize
		if to > currentLength {
			to = currentLength
		}
		partNumber++

		rangeStr := fmt.Sprintf("bytes=%d-%d", start, to-1)
		limiter <- struct{}{}
		go func(partNumber int, rangeStr string) {
			_, part, err := multi.PutPartCopyWithContentLength(partNumber,
				CopyOptions{CopySourceOptions: rangeStr},
				sourcePathForCopy, currentLength)
			if err == nil {
				completedParts[partNumber-1] = part
			} else {
				log.Printf("Unable in PutPartCopy of part %d for %s: %v\n", partNumber, sourcePathForCopy, err)
			}
			errChan <- err
			<-limiter
		}(partNumber, rangeStr)
	}

	fullyCompleted := true
	for range completedParts {
		err := <-errChan
		if err != nil {
			fullyCompleted = false
		}
	}

	if fullyCompleted {
		err = multi.Complete(completedParts)
	} else {
		err = multi.Abort()
	}

	return err
}
905727
func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))

	// Read metadata associated with the object from all disks.
	storageDisks := xl.getDisks()

	metaArr, errs := readAllXLMetadata(ctx, storageDisks, srcBucket, srcObject)

	// get Quorum for this object
	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
	if err != nil {
		return oi, toObjectErr(err, srcBucket, srcObject)
	}

	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
		return oi, toObjectErr(reducedErr, srcBucket, srcObject)
	}

	// List all online disks.
	_, modTime := listOnlineDisks(storageDisks, metaArr, errs)

	// Pick latest valid metadata.
	xlMeta, err := pickValidXLMeta(ctx, metaArr, modTime, readQuorum)
	if err != nil {
		return oi, toObjectErr(err, srcBucket, srcObject)
	}

	// Length of the file to read.
	length := xlMeta.Stat.Size

	// Check if this request is only metadata update.
	if cpSrcDstSame {
		// Update `xl.json` content on each disks.
		for index := range metaArr {
			metaArr[index].Meta = srcInfo.UserDefined
			metaArr[index].Meta["etag"] = srcInfo.ETag
		}

		var onlineDisks []StorageAPI

		tempObj := mustGetUUID()

		// Cleanup in case of xl.json writing failure
		defer xl.deleteObject(ctx, minioMetaTmpBucket, tempObj, writeQuorum, false)

		// Write unique `xl.json` for each disk.
		if onlineDisks, err = writeUniqueXLMetadata(ctx, storageDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
			return oi, toObjectErr(err, srcBucket, srcObject)
		}
		// Rename atomically `xl.json` from tmp location to destination for each disk.
		if _, err = renameXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempObj, srcBucket, srcObject, writeQuorum); err != nil {
			return oi, toObjectErr(err, srcBucket, srcObject)
		}
		return xlMeta.ToObjectInfo(srcBucket, srcObject), nil
	}

	// Initialize pipe.
	pipeReader, pipeWriter := io.Pipe()

	go func() {
		var startOffset int64 // Read the whole file.
		if gerr := xl.getObject(ctx, srcBucket, srcObject, startOffset, length, pipeWriter, srcInfo.ETag, srcOpts); gerr != nil {
			pipeWriter.CloseWithError(toObjectErr(gerr, srcBucket, srcObject))
			return
		}
		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
	}()

	hashReader, err := hash.NewReader(pipeReader, length, "", "", length)
	if err != nil {
		logger.LogIf(ctx, err)
		return oi, toObjectErr(err, dstBucket, dstObject)
	}
	putOpts := ObjectOptions{UserDefined: srcInfo.UserDefined, ServerSideEncryption: dstOpts.ServerSideEncryption}
	objInfo, err := xl.putObject(ctx, dstBucket, dstObject, NewPutObjReader(hashReader, nil, nil), putOpts)
	if err != nil {
		return oi, toObjectErr(err, dstBucket, dstObject)
	}

	// Explicitly close the reader.
	pipeReader.Close()

	return objInfo, nil
}
1200052
func (s *Bucket) CompleteMultipartUpload(objectKey string, input *CompleteMultipartUploadInput) (*CompleteMultipartUploadOutput, error) {
	r, x, err := s.CompleteMultipartUploadRequest(objectKey, input)

	if err != nil {
		return x, err
	}

	err = r.Send()
	if err != nil {
		return nil, err
	}

	requestID := r.HTTPResponse.Header.Get(http.CanonicalHeaderKey("X-QS-Request-ID"))
	x.RequestID = &requestID

	return x, err
}
846067
func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, s, d minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
	cpSrcDstSame := strings.EqualFold(path.Join(srcBucket, srcObject), path.Join(dstBucket, dstObject))
	if cpSrcDstSame {
		var gwMeta gwMetaV1
		if s.ServerSideEncryption != nil && d.ServerSideEncryption != nil &&
			((s.ServerSideEncryption.Type() == encrypt.SSEC && d.ServerSideEncryption.Type() == encrypt.SSEC) ||
				(s.ServerSideEncryption.Type() == encrypt.S3 && d.ServerSideEncryption.Type() == encrypt.S3)) {
			gwMeta, err = l.getGWMetadata(ctx, srcBucket, getDareMetaPath(srcObject))
			if err != nil {
				return
			}
			header := make(http.Header)
			if d.ServerSideEncryption != nil {
				d.ServerSideEncryption.Marshal(header)
			}
			for k, v := range header {
				srcInfo.UserDefined[k] = v[0]
			}
			gwMeta.Meta = srcInfo.UserDefined
			if err = l.writeGWMetadata(ctx, dstBucket, getDareMetaPath(dstObject), gwMeta, minio.ObjectOptions{}); err != nil {
				return objInfo, minio.ErrorRespToObjectError(err)
			}
			return gwMeta.ToObjectInfo(dstBucket, dstObject), nil
		}
	}
	dstOpts := minio.ObjectOptions{ServerSideEncryption: d.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
	return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, dstOpts)
}
1243126
func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (string, error) {
	if err := checkNewMultipartArgs(ctx, bucket, object, xl); err != nil {
		return "", err
	}
	// No metadata is set, allocate a new one.
	if opts.UserDefined == nil {
		opts.UserDefined = make(map[string]string)
	}
	return xl.newMultipartUpload(ctx, bucket, object, opts.UserDefined)
}
1112724
func (c *container) multipartUpload(name string, r io.Reader, size int64) error {
	chunkSize, err := determineChunkSize(size)
	if err != nil {
		return err
	}
	var buf = make([]byte, chunkSize)

	var blocks []az.Block
	var rawID uint64
	blob := c.client.GetContainerReference(c.id).GetBlobReference(name)

	// TODO: upload the parts in parallel
	for {
		n, err := r.Read(buf)
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}

		blockID := encodedBlockID(rawID)
		chunk := buf[:n]

		if err := blob.PutBlock(blockID, chunk, nil); err != nil {
			return err
		}

		blocks = append(blocks, az.Block{
			ID:     blockID,
			Status: az.BlockStatusLatest,
		})
		rawID++
	}

	return blob.PutBlockList(blocks, nil)
}