Finalize Block blob
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param dict metadata: metadata dict
176463
def _finalize_block_blob(self, ud, metadata):
        # type: (Uploader, blobxfer.models.upload.Descriptor, dict) -> None
        
        if not ud.entity.is_encrypted and ud.must_compute_md5:
            digest = blobxfer.util.base64_encode_as_string(ud.md5.digest())
        else:
            digest = None
        blobxfer.operations.azure.blob.block.put_block_list(
            ud.entity, ud.last_block_num, digest, metadata)
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                blobxfer.operations.azure.blob.block.put_block_list(
                    ase, ud.last_block_num, digest, metadata)
825803
def _initialize_integrity_checkers(self, options):
        # type: (Descriptor, blobxfer.models.options.Upload) -> None
        
        if self._ase.is_encrypted:
            # ensure symmetric key exists
            if blobxfer.util.is_none_or_empty(
                    self._ase.encryption_metadata.symmetric_key):
                raise RuntimeError(
                    ('symmetric key is invalid: provide RSA private key '
                     'or metadata corrupt for {}').format(
                         self.local_path.absolute_path))
            self.hmac = self._ase.encryption_metadata.initialize_hmac()
        # both hmac and md5 can be enabled
        if (options.store_file_properties.md5 and
                not self.remote_is_append_blob):
            self.md5 = blobxfer.util.new_md5_hasher()
705898
def upload_metadata_cli(directory, create_csv='', review='',
                        max_size='128m', verbose=False, debug=False):
    
    return upload_metadata(directory, create_csv, review,
                           max_size, verbose, debug)
833996
def _set_blob_properties(self, ud):
        # type: (Uploader, blobxfer.models.upload.Descriptor) -> None
        
        if ud.requires_non_encrypted_md5_put:
            digest = blobxfer.util.base64_encode_as_string(ud.md5.digest())
        else:
            digest = None
        blobxfer.operations.azure.blob.set_blob_properties(ud.entity, digest)
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                blobxfer.operations.azure.blob.set_blob_properties(ase, digest)
710202
def parse(self, stream, media_type=None, parser_context=None):
        

        parser_context = parser_context or {}
        request = parser_context['request']
        encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)
        meta = request.META
        upload_handlers = request.upload_handlers
        filename = self.get_filename(stream, media_type, parser_context)

        # Note that this code is extracted from Django's handling of
        # file uploads in MultiPartParser.
        content_type = meta.get('HTTP_CONTENT_TYPE',
                                meta.get('CONTENT_TYPE', ''))
        try:
            content_length = int(meta.get('HTTP_CONTENT_LENGTH',
                                          meta.get('CONTENT_LENGTH', 0)))
        except (ValueError, TypeError):
            content_length = None

        # See if the handler will want to take care of the parsing.
        for handler in upload_handlers:
            result = handler.handle_raw_input(None,
                                              meta,
                                              content_length,
                                              None,
                                              encoding)
            if result is not None:
                return DataAndFiles({}, {'file': result[1]})

        # This is the standard case.
        possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size]
        chunk_size = min([2 ** 31 - 4] + possible_sizes)
        chunks = ChunkIter(stream, chunk_size)
        counters = [0] * len(upload_handlers)

        for index, handler in enumerate(upload_handlers):
            try:
                handler.new_file(None, filename, content_type,
                                 content_length, encoding)
            except StopFutureHandlers:
                upload_handlers = upload_handlers[:index + 1]
                break

        for chunk in chunks:
            for index, handler in enumerate(upload_handlers):
                chunk_length = len(chunk)
                chunk = handler.receive_data_chunk(chunk, counters[index])
                counters[index] += chunk_length
                if chunk is None:
                    break

        for index, handler in enumerate(upload_handlers):
            file_obj = handler.file_complete(counters[index])
            if file_obj:
                return DataAndFiles({}, {'file': file_obj})
        raise ParseError("FileUpload parse error - "
                         "none of upload handlers can handle the stream")
881392
def finalize_integrity(self):
        # type: (Descriptor) -> None
        
        with self._meta_lock:
            if self._finalized:
                return
        # check final file integrity
        check = False
        msg = None
        if self.hmac is not None:
            mac = self._ase.encryption_metadata.encryption_authentication.\
                message_authentication_code
            digest = blobxfer.util.base64_encode_as_string(self.hmac.digest())
            if digest == mac:
                check = True
            msg = '{}: {}, {} {} <L..R> {}'.format(
                self._ase.encryption_metadata.encryption_authentication.
                algorithm,
                'OK' if check else 'MISMATCH',
                self._ase.path,
                digest,
                mac,
            )
        elif self.md5 is not None:
            digest = blobxfer.util.base64_encode_as_string(self.md5.digest())
            if digest == self._ase.md5:
                check = True
            msg = 'MD5: {}, {} {} <L..R> {}'.format(
                'OK' if check else 'MISMATCH',
                self._ase.path,
                digest,
                self._ase.md5,
            )
        else:
            check = True
            msg = 'MD5: SKIPPED, {} None <L..R> {}'.format(
                self._ase.path,
                self._ase.md5
            )
        # cleanup if download failed
        if not check:
            self._integrity_failed = True
            logger.error(msg)
        else:
            logger.info(msg)
1083060
def _remove_incomplete_upload(self, bucket_name, object_name, upload_id):
        

        # No reason to store successful response, for errors
        # relevant exceptions are thrown.
        self._url_open('DELETE', bucket_name=bucket_name,
                       object_name=object_name, query={'uploadId': upload_id},
                       headers={})
425907
def _finalize_azure_file(self, ud, metadata):
        # type: (Uploader, blobxfer.models.upload.Descriptor, dict) -> None
        
        # set md5 file property if required
        if ud.requires_non_encrypted_md5_put:
            digest = blobxfer.util.base64_encode_as_string(ud.md5.digest())
        else:
            digest = None
        if digest is not None or ud.entity.cache_control is not None:
            blobxfer.operations.azure.file.set_file_properties(
                ud.entity, digest)
            if blobxfer.util.is_not_empty(ud.entity.replica_targets):
                for ase in ud.entity.replica_targets:
                    blobxfer.operations.azure.file.set_file_properties(
                        ase, digest)
        # set file metadata if needed
        if blobxfer.util.is_not_empty(metadata):
            blobxfer.operations.azure.file.set_file_metadata(
                ud.entity, metadata)
            if blobxfer.util.is_not_empty(ud.entity.replica_targets):
                for ase in ud.entity.replica_targets:
                    blobxfer.operations.azure.file.set_file_metadata(
                        ase, metadata)
972374
def fileattr_from_metadata(md):
    # type: (dict) -> collections.namedtuple
    
    try:
        mdattr = json.loads(
            md[JSON_KEY_BLOBXFER_METADATA])[_JSON_KEY_FILE_ATTRIBUTES]
    except (KeyError, TypeError):
        return None
    else:
        if blobxfer.util.on_windows():
            global _FILEATTR_WARNED_ON_WINDOWS
            if not _FILEATTR_WARNED_ON_WINDOWS:
                _FILEATTR_WARNED_ON_WINDOWS = True
                logger.warning(
                    'file attributes store/restore on Windows is not '
                    'supported yet')
            fileattr = None
        else:
            try:
                fileattr = PosixFileAttr(
                    mode=mdattr[_JSON_KEY_FILE_ATTRIBUTES_POSIX][
                        _JSON_KEY_FILE_ATTRIBUTES_MODE],
                    uid=mdattr[_JSON_KEY_FILE_ATTRIBUTES_POSIX][
                        _JSON_KEY_FILE_ATTRIBUTES_UID],
                    gid=mdattr[_JSON_KEY_FILE_ATTRIBUTES_POSIX][
                        _JSON_KEY_FILE_ATTRIBUTES_GID],
                )
            except KeyError:
                fileattr = None
        return fileattr
1308379
def create_object(self, api, metadata=None):
        
        metadata = {k: v for k, v in (metadata or {}).items()}
        metadata.setdefault('creator', '{0.__name__} {0.__version__}'.format(pycdstar))
        metadata.setdefault('path', '%s' % self.path)
        metadata.update(self.add_metadata())
        bitstream_specs = [self] + self.add_bitstreams()
        obj = api.get_object()
        res = {}
        try:
            obj.metadata = metadata
            for file_ in bitstream_specs:
                res[file_.bitstream_type] = file_.add_as_bitstream(obj)
        except:  # noqa: E722
            obj.delete()
            raise
        return obj, metadata, res
305275
def upload_delete(self, token, **kwargs):
        ""
        api_path = "/api/v2/uploads/{token}.json"
        api_path = api_path.format(token=token)
        return self.call(api_path, method="DELETE", **kwargs)
1059399
def multipart_upload_lister(bucket, key_marker='',
                            upload_id_marker='',
                            headers=None):
    
    more_results = True
    k = None
    while more_results:
        rs = bucket.get_all_multipart_uploads(key_marker=key_marker,
                                              upload_id_marker=upload_id_marker,
                                              headers=headers)
        for k in rs:
            yield k
        key_marker = rs.next_key_marker
        upload_id_marker = rs.next_upload_id_marker
        more_results= rs.is_truncated
307889
def _finalize_metadata(self, node):
        
        final = {}
        for key, val in iter(node.metadata.items()):
            #val = list(val)
            #if isinstance(val[0], tuple):
            #    val = [dict(v) for v in val]
            final[key] = list(val)
        node.metadata = final
        return node
605139
private void initMultiPartUpload() throws IOException {
    // Generate the object metadata by setting server side encryption, md5 checksum,
    // and encoding as octet stream since no assumptions are made about the file type
    ObjectMetadata meta = new ObjectMetadata();
    if (mSseEnabled) {
      meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    }
    if (mHash != null) {
      meta.setContentMD5(Base64.encodeAsString(mHash.digest()));
    }
    meta.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM);

    AmazonClientException lastException;
    InitiateMultipartUploadRequest initRequest =
        new InitiateMultipartUploadRequest(mBucketName, mKey).withObjectMetadata(meta);
    do {
      try {
        mUploadId = mClient.initiateMultipartUpload(initRequest).getUploadId();
        return;
      } catch (AmazonClientException e) {
        lastException = e;
      }
    } while (mRetryPolicy.attempt());
    // This point is only reached if the operation failed more
    // than the allowed retry count
    throw new IOException("Unable to init multipart upload to " + mKey, lastException);
  }
1176488
def _process_download_descriptor(self, dd):
        # type: (Downloader, blobxfer.models.download.Descriptor) -> None
        
        # update progress bar
        self._update_progress_bar()
        # get download offsets
        offsets, resume_bytes = dd.next_offsets()
        # add resume bytes to counter
        if resume_bytes is not None:
            with self._disk_operation_lock:
                self._download_bytes_sofar += resume_bytes
                logger.debug('adding {} sofar {} from {}'.format(
                    resume_bytes, self._download_bytes_sofar, dd.entity.name))
            del resume_bytes
        # check if all operations completed
        if offsets is None and dd.all_operations_completed:
            finalize = True
            sfpath = str(dd.final_path)
            # finalize integrity
            dd.finalize_integrity()
            # vectored io checks
            if dd.entity.vectored_io is not None:
                with self._transfer_lock:
                    if sfpath not in self._vio_map:
                        self._vio_map[sfpath] = 1
                    else:
                        self._vio_map[sfpath] += 1
                    if (self._vio_map[sfpath] ==
                            dd.entity.vectored_io.total_slices):
                        self._vio_map.pop(sfpath)
                    else:
                        finalize = False
            # finalize file
            if finalize:
                dd.finalize_file()
            # accounting
            with self._transfer_lock:
                self._download_sofar += 1
                if dd.entity.is_encrypted:
                    self._dd_map.pop(sfpath)
                self._transfer_set.remove(
                    blobxfer.operations.download.Downloader.
                    create_unique_transfer_operation_id(dd.entity))
                self._transfer_cc.pop(dd.entity.path, None)
            return
        # re-enqueue for other threads to download
        if offsets is None:
            self._transfer_queue.put(dd)
            return
        # ensure forthcoming disk operation is accounted for
        with self._disk_operation_lock:
            self._disk_set.add(
                blobxfer.operations.download.Downloader.
                create_unique_disk_operation_id(dd, offsets))
        # check if there are too many concurrent connections
        with self._transfer_lock:
            self._transfer_cc[dd.entity.path] += 1
            cc_xfer = self._transfer_cc[dd.entity.path]
        if cc_xfer <= self._spec.options.max_single_object_concurrency:
            self._transfer_queue.put(dd)
        # issue get range
        if dd.entity.mode == blobxfer.models.azure.StorageModes.File:
            data = blobxfer.operations.azure.file.get_file_range(
                dd.entity, offsets)
        else:
            data = blobxfer.operations.azure.blob.get_blob_range(
                dd.entity, offsets)
        with self._transfer_lock:
            self._transfer_cc[dd.entity.path] -= 1
        if cc_xfer > self._spec.options.max_single_object_concurrency:
            self._transfer_queue.put(dd)
        # enqueue data for processing
        self._disk_queue.put((dd, offsets, data))
631704
def handle_upload(repo, **kwargs):
    
    log.info('upload: %s' %(repo))
    name = kwargs.get('filename', 'Untitled')
    data = kwargs.get('filedata')
    repo.addItem(Item.from_string(repo=repo, name=name, string=data))
1060030
def _process_synccopy_descriptor(self, sd):
        # type: (SyncCopy, blobxfer.models.synccopy.Descriptor) -> None
        
        # update progress bar
        self._update_progress_bar()
        # get download offsets
        offsets, resume_bytes = sd.next_offsets()
        # add resume bytes to counter
        if resume_bytes is not None:
            with self._transfer_lock:
                self._synccopy_bytes_sofar += resume_bytes
                logger.debug('adding {} sofar {} from {}'.format(
                    resume_bytes, self._synccopy_bytes_sofar,
                    sd.dst_entity.name))
            del resume_bytes
        # check if all operations completed
        if offsets is None and sd.all_operations_completed:
            # finalize upload for non-one shots
            if not sd.is_one_shot_block_blob:
                self._finalize_upload(sd)
            else:
                # set access tier for one shots
                if sd.requires_access_tier_set:
                    blobxfer.operations.azure.blob.block.set_blob_access_tier(
                        sd.dst_entity)
            # accounting
            with self._transfer_lock:
                self._transfer_set.remove(
                    blobxfer.operations.synccopy.SyncCopy.
                    create_unique_transfer_operation_id(
                        sd.src_entity, sd.dst_entity))
                self._synccopy_sofar += 1
            return
        # re-enqueue for finalization if no offsets
        if offsets is None:
            self._transfer_queue.put(sd)
            return
        # prepare upload
        if offsets.chunk_num == 0:
            self._prepare_upload(sd.dst_entity)
        # prepare replica targets
        if blobxfer.util.is_not_empty(sd.dst_entity.replica_targets):
            for ase in sd.dst_entity.replica_targets:
                if offsets.chunk_num == 0:
                    self._prepare_upload(ase)
        # re-enqueue for other threads to download next offset if not append
        if sd.src_entity.mode != blobxfer.models.azure.StorageModes.Append:
            self._transfer_queue.put(sd)
        # issue get range
        if sd.src_entity.mode == blobxfer.models.azure.StorageModes.File:
            data = blobxfer.operations.azure.file.get_file_range(
                sd.src_entity, offsets)
        else:
            data = blobxfer.operations.azure.blob.get_blob_range(
                sd.src_entity, offsets)
        # process data for upload
        self._process_data(sd, sd.dst_entity, offsets, data)
        # iterate replicas
        if blobxfer.util.is_not_empty(sd.dst_entity.replica_targets):
            for ase in sd.dst_entity.replica_targets:
                self._process_data(sd, ase, offsets, data)
        # re-enqueue for append blobs
        if sd.src_entity.mode == blobxfer.models.azure.StorageModes.Append:
            self._transfer_queue.put(sd)