Prepare upload
        :param Uploader self: this
        :param blobxfer.models.azure.StorageEntity ase: Storage entity
509889
def _prepare_upload(self, ase):
        # type: (Uploader, blobxfer.models.azure.StorageEntity) -> None
        
        if ase.mode == blobxfer.models.azure.StorageModes.Append:
            if ase.append_create:
                # create container if necessary
                blobxfer.operations.azure.blob.create_container(
                    ase, self._containers_created)
                # create remote blob
                blobxfer.operations.azure.blob.append.create_blob(ase)
        elif ase.mode == blobxfer.models.azure.StorageModes.Block:
            # create container if necessary
            blobxfer.operations.azure.blob.create_container(
                ase, self._containers_created)
        elif ase.mode == blobxfer.models.azure.StorageModes.File:
            # create share directory structure
            with self._fileshare_dir_lock:
                # create container if necessary
                blobxfer.operations.azure.file.create_share(
                    ase, self._containers_created)
                # create parent directories
                blobxfer.operations.azure.file.create_all_parent_directories(
                    ase, self._dirs_created)
            # create remote file
            blobxfer.operations.azure.file.create_file(ase)
        elif ase.mode == blobxfer.models.azure.StorageModes.Page:
            # create container if necessary
            blobxfer.operations.azure.blob.create_container(
                ase, self._containers_created)
            # create remote blob
            blobxfer.operations.azure.blob.page.create_blob(ase)
129077
def _set_blob_metadata(self, ud, metadata):
        # type: (Uploader, blobxfer.models.upload.Descriptor, dict) -> None
        
        blobxfer.operations.azure.blob.set_blob_metadata(ud.entity, metadata)
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                blobxfer.operations.azure.blob.set_blob_metadata(ase, metadata)
183134
def _uploadStream(self, jobStoreFileID, container, checkForModification=False, encrypted=None):
        
        if checkForModification:
            try:
                expectedVersion = container.get_blob_properties(blob_name=str(jobStoreFileID)).properties.etag
            except AzureMissingResourceHttpError:
                expectedVersion = None

        if encrypted is None:
            encrypted = self.keyPath is not None
        elif encrypted:
            if self.keyPath is None:
                raise RuntimeError('Encryption requested but no key was provided')

        maxBlockSize = self._maxAzureBlockBytes
        if encrypted:
            # There is a small overhead for encrypted data.
            maxBlockSize -= encryption.overhead

        store = self

        class UploadPipe(WritablePipe):

            def readFrom(self, readable):
                blocks = []
                try:
                    while True:
                        buf = readable.read(maxBlockSize)
                        if len(buf) == 0:
                            # We're safe to break here even if we never read anything, since
                            # putting an empty block list creates an empty blob.
                            break
                        if encrypted:
                            buf = encryption.encrypt(buf, store.keyPath)
                        blockID = store._newFileID()
                        container.put_block(blob_name=str(jobStoreFileID),
                                            block=buf,
                                            block_id=blockID)
                        blocks.append(BlobBlock(blockID))
                except:
                    with panic(log=logger):
                        # This is guaranteed to delete any uncommitted blocks.
                        container.delete_blob(blob_name=str(jobStoreFileID))

                if checkForModification and expectedVersion is not None:
                    # Acquire a (60-second) write lock,
                    leaseID = container.acquire_blob_lease(blob_name=str(jobStoreFileID),
                                                           lease_duration=60)
                    # check for modification,
                    blob = container.get_blob_properties(blob_name=str(jobStoreFileID))
                    if blob.properties.etag != expectedVersion:
                        container.release_blob_lease(blob_name=str(jobStoreFileID), lease_id=leaseID)
                        raise ConcurrentFileModificationException(jobStoreFileID)
                    # commit the file,
                    container.put_block_list(blob_name=str(jobStoreFileID),
                                             block_list=blocks,
                                             lease_id=leaseID,
                                             metadata=dict(encrypted=str(encrypted)))
                    # then release the lock.
                    container.release_blob_lease(blob_name=str(jobStoreFileID), lease_id=leaseID)
                else:
                    # No need to check for modification, just blindly write over whatever
                    # was there.
                    container.put_block_list(blob_name=str(jobStoreFileID),
                                             block_list=blocks,
                                             metadata=dict(encrypted=str(encrypted)))

        with UploadPipe() as writable:
            yield writable
848881
def _generate_destination_for_source(self, local_path):
        # type: (Uploader, blobxfer.models.upload.LocalSourcePath) ->
        #        Tuple[blobxfer.operations.azure.StorageAccount,
        #        blobxfer.models.azure.StorageEntity)
        
        # construct stripped destination path
        spath = local_path.relative_path
        # apply strip components
        if self._spec.options.strip_components > 0:
            _rparts = local_path.relative_path.parts
            _strip = min(
                (len(_rparts) - 1, self._spec.options.strip_components)
            )
            if _strip > 0:
                spath = pathlib.Path(*_rparts[_strip:])
        # create a storage entity for each destination
        for sa, cont, name, dpath in self._get_destination_paths():
            # if not renaming, form name from with spath
            if not self._spec.options.rename and not local_path.use_stdin:
                name = str(name / spath)
            if blobxfer.util.is_none_or_empty(name):
                raise ValueError(
                    ('invalid destination, must specify a container or '
                     'fileshare and remote file name: {}').format(dpath))
            # do not check for existing remote right now if striped
            # vectored io mode
            if (self._spec.options.vectored_io.distribution_mode ==
                    blobxfer.models.upload.
                    VectoredIoDistributionMode.Stripe):
                ase = None
            else:
                if sa.can_read_object:
                    ase = self._check_for_existing_remote(sa, cont, name)
                else:
                    ase = None
            if ase is None:
                # encryption metadata will be populated later, if required
                ase = blobxfer.models.azure.StorageEntity(cont, ed=None)
                ase.populate_from_local(
                    sa, cont, name, self._spec.options.mode,
                    self._spec.options.store_file_properties.cache_control)
                if ase.mode == blobxfer.models.azure.StorageModes.Block:
                    ase.access_tier = self._spec.options.access_tier
            yield sa, ase
1100305
def __upload(self, resource, bytes):
        """"""

        # note: string conversion required here due to open encoding bug in requests-oauthlib.
        headers = {
            'x-ton-expires': http_time(self.options.get('x-ton-expires', self._DEFAULT_EXPIRE)),
            'content-length': str(self._file_size),
            'content-type': self.content_type
        }

        return Request(self._client, 'post', resource,
                       domain=self._DEFAULT_DOMAIN, headers=headers, body=bytes).perform()
620443
def add_entity(self, entity):
        
        self._ensure_loaded()
        self.entities[str(entity)] = entity
723947
def upload_create(self, data, filename=None, token=None, **kwargs):
        ""
        api_path = "/api/v2/uploads.json"
        api_query = {}
        if "query" in kwargs.keys():
            api_query.update(kwargs["query"])
            del kwargs["query"]
        if filename:
            api_query.update({
                "filename": filename,
            })
        if token:
            api_query.update({
                "token": token,
            })
        return self.call(api_path, query=api_query, method="POST", data=data, **kwargs)
762285
def _generate_destination_for_source(self, src_ase):
        # type: (SyncCopy, blobxfer.models.azure.StorageEntity) ->
        #        blobxfer.models.azure.StorageEntity)
        
        # create a storage entity for each destination
        for sa, cont, name, dpath in self._get_destination_paths():
            if self._spec.options.rename:
                name = str(pathlib.Path(name))
                if name == '.':
                    raise RuntimeError(
                        'attempting rename multiple files to a directory')
            else:
                name = str(pathlib.Path(name) / src_ase.name)
            # translate source mode to dest mode
            dst_mode = self._translate_src_mode_to_dst_mode(src_ase.mode)
            dst_ase = self._check_for_existing_remote(sa, cont, name, dst_mode)
            if dst_ase is None:
                dst_ase = blobxfer.models.azure.StorageEntity(cont, ed=None)
                dst_ase.populate_from_local(
                    sa, cont, name, dst_mode, src_ase.cache_control)
                dst_ase.size = src_ase.size
            # overwrite tier with specified storage tier
            if (dst_mode == blobxfer.models.azure.StorageModes.Block and
                    self._spec.options.access_tier is not None):
                dst_ase.access_tier = self._spec.options.access_tier
            # check condition for dst
            action = self._check_copy_conditions(src_ase, dst_ase)
            if action == SynccopyAction.Copy:
                yield dst_ase
            elif action == SynccopyAction.Skip:
                # add to exclusion set if skipping
                if self._spec.options.delete_extraneous_destination:
                    uid = (
                        blobxfer.operations.synccopy.SyncCopy.
                        create_deletion_id(
                            dst_ase._client, dst_ase.container, dst_ase.name)
                    )
                    self._delete_exclude.add(uid)
                if self._general_options.dry_run:
                    logger.info('[DRY RUN] skipping: {} -> {}'.format(
                        src_ase.path, dst_ase.path))
366155
public function initializeUploadAssetAction()
    {
        $propertyMappingConfiguration = $this->arguments->getArgument('asset')->getPropertyMappingConfiguration();
        $propertyMappingConfiguration->allowAllProperties();
        $propertyMappingConfiguration->setTypeConverterOption(PersistentObjectConverter::class, PersistentObjectConverter::CONFIGURATION_CREATION_ALLOWED, true);
        $propertyMappingConfiguration->setTypeConverterOption(AssetInterfaceConverter::class, AssetInterfaceConverter::CONFIGURATION_ONE_PER_RESOURCE, true);
        $propertyMappingConfiguration->allowCreationForSubProperty('resource');
    }
79906
def _run(self):
        # type: (Uploader) -> None
        
        # mark start
        self._start_time = blobxfer.util.datetime_now()
        logger.info('blobxfer start time: {0}'.format(self._start_time))
        # check renames
        if not self._spec.sources.can_rename() and self._spec.options.rename:
            raise RuntimeError(
                'cannot rename to specified destination with multiple sources')
        # initialize resume db if specified
        if self._general_options.resume_file is not None:
            self._resume = blobxfer.operations.resume.UploadResumeManager(
                self._general_options.resume_file)
        # initialize MD5 processes
        if ((self._spec.options.store_file_properties.md5 or
             self._spec.skip_on.md5_match) and
                self._general_options.concurrency.md5_processes > 0):
            self._md5_offload = blobxfer.operations.md5.LocalFileMd5Offload(
                num_workers=self._general_options.concurrency.md5_processes)
            self._md5_offload.initialize_check_thread(
                self._check_for_uploads_from_md5)
        # initialize crypto processes
        if (self._spec.options.rsa_public_key is not None and
                self._general_options.concurrency.crypto_processes > 0):
            logger.warning(
                'crypto offload for upload is not possible due to '
                'sequential nature of {} and FullBlob encryption mode'.format(
                    blobxfer.models.crypto.EncryptionMetadata.
                    _ENCRYPTION_ALGORITHM)
            )
        # initialize worker threads
        self._initialize_disk_threads()
        self._initialize_transfer_threads()
        # initialize local counters
        files_processed = 0
        skipped_files = 0
        skipped_size = 0
        approx_total_bytes = 0
        # iterate through source paths to upload
        seen = set()
        for src in self._spec.sources.files(self._general_options.dry_run):
            # create a destination array for the source
            dest = [
                (sa, ase) for sa, ase in
                self._generate_destination_for_source(src)
            ]
            for action, lp, ase in self._vectorize_and_bind(src, dest):
                dest_id = blobxfer.operations.upload.Uploader.\
                    create_destination_id(ase._client, ase.container, ase.name)
                if dest_id in seen:
                    raise RuntimeError(
                        'duplicate destination entity detected: {}/{}'.format(
                            ase._client.primary_endpoint, ase.path))
                seen.add(dest_id)
                if self._spec.options.delete_extraneous_destination:
                    self._delete_exclude.add(dest_id)
                files_processed += 1
                if action == UploadAction.Skip:
                    skipped_files += 1
                    skipped_size += ase.size if ase.size is not None else 0
                    if self._general_options.dry_run:
                        logger.info('[DRY RUN] skipping: {} -> {}'.format(
                            lp.absolute_path, ase.path))
                    continue
                approx_total_bytes += lp.size
                if blobxfer.util.is_not_empty(ase.replica_targets):
                    approx_total_bytes += lp.size * len(ase.replica_targets)
                # add to potential upload set
                uid = blobxfer.operations.upload.Uploader.create_unique_id(
                    lp, ase)
                with self._upload_lock:
                    self._upload_set.add(uid)
                    self._upload_total += 1
                if action == UploadAction.CheckMd5:
                    self._pre_md5_skip_on_check(lp, ase)
                elif action == UploadAction.Upload:
                    if self._general_options.dry_run:
                        logger.info('[DRY RUN] upload: {} -> {}'.format(
                            lp.absolute_path, ase.path))
                        with self._upload_lock:
                            self._upload_set.remove(uid)
                            self._upload_total -= 1
                    else:
                        self._add_to_upload_queue(lp, ase, uid)
        del seen
        # set remote files processed
        with self._md5_meta_lock:
            self._all_files_processed = True
        with self._upload_lock:
            upload_size_mib = approx_total_bytes / blobxfer.util.MEGABYTE
            logger.debug(
                ('{0} files {1:.4f} MiB filesize and/or lmt_ge '
                 'skipped').format(
                    skipped_files, skipped_size / blobxfer.util.MEGABYTE))
            logger.debug(
                ('{0} local files processed, waiting for upload '
                 'completion of approx. {1:.4f} MiB').format(
                     files_processed, upload_size_mib))
        del files_processed
        del skipped_files
        del skipped_size
        del upload_size_mib
        del approx_total_bytes
        # wait for uploads to complete
        self._wait_for_disk_threads(terminate=False)
        self._wait_for_transfer_threads(terminate=False)
        end_time = blobxfer.util.datetime_now()
        # update progress bar
        self._update_progress_bar()
        # check for exceptions
        if len(self._exceptions) > 0:
            logger.error('exceptions encountered while uploading')
            # raise the first one
            raise self._exceptions[0]
        # check for mismatches
        if (self._upload_sofar != self._upload_total or
                self._upload_bytes_sofar != self._upload_bytes_total):
            raise RuntimeError(
                'upload mismatch: [count={}/{} bytes={}/{}]'.format(
                    self._upload_sofar, self._upload_total,
                    self._upload_bytes_sofar, self._upload_bytes_total))
        # delete all remaining local files not accounted for if
        # delete extraneous enabled
        self._delete_extraneous_files()
        # delete resume file if we've gotten this far
        if self._resume is not None:
            self._resume.delete()
        # output throughput
        if self._upload_start_time is not None:
            ultime = (end_time - self._upload_start_time).total_seconds()
            if ultime == 0:  # noqa
                ultime = 1e-9
            mibup = self._upload_bytes_total / blobxfer.util.MEGABYTE
            mibps = mibup / ultime
            logger.info(
                ('elapsed upload + verify time and throughput of {0:.4f} '
                 'GiB: {1:.3f} sec, {2:.4f} Mbps ({3:.3f} MiB/s)').format(
                     mibup / 1024, ultime, mibps * 8, mibps))
        end_time = blobxfer.util.datetime_now()
        logger.info('blobxfer end time: {0} (elapsed: {1:.3f} sec)'.format(
            end_time, (end_time - self._start_time).total_seconds()))
1229504
def deserialize_upload(value, url):
    
    result = {'name': None, 'storage': None}
    try:
        result = signing.loads(value, salt=url)
    except signing.BadSignature:
        # TODO: Log invalid signature
        pass
    else:
        try:
            result['storage'] = get_storage_class(result['storage'])
        except (ImproperlyConfigured, ImportError):
            # TODO: Log invalid class
            result = {'name': None, 'storage': None}
    return result
1227433
def configure_upload(graph, ns, mappings, exclude_func=None):
    
    convention = UploadConvention(graph, exclude_func)
    convention.configure(ns, mappings)
757787
def upload(file)
      # get the path for upload
      response = Atlas.client.get("#{url_builder.box_provider_url}/upload")

      # upload the file
      upload_url = response['upload_path']
      Excon.put(upload_url, body: file)
    end
484845
function Uploadfs() {
  var tempPath, imageSizes;
  var scaledJpegQuality;
  var self = this;
  /**
   * Initialize uploadfs. The init method passes options to the backend and invokes a callback when the backend is ready.
   * @param  {Object}   options: backend, imageSizes, orientOriginals, tempPath, copyOriginal, scaledJpegQuality, contentType, cdn. backend is the only mandatory option. See the README and individual methods for details.
   * @param  {Object}   options.cdn               - An object, that defines cdn settings
   * @param  {Boolean}  options.cdn.enabled=true  - Whether the cdn should be anbled or not
   * @param  {String}   options.cdn.url           - The cdn-url
   * @param  {Function} callback                  - Will receive the usual err argument
   */
  self.init = function (options, callback) {
    self.options = options;
    self.prefix = self.options.prefix || '';
    // bc: support options.backend
    self._storage = options.storage || options.backend;
    if (!self._storage) {
      return callback("Storage backend must be specified");
    }
    // Load standard storage backends, by name. You can also pass an object
    // with your own implementation
    if (typeof (self._storage) === 'string') {
      self._storage = require('./lib/storage/' + self._storage + '.js')();
    }

    // If you want to deliver your images
    // over a CDN then this could be set in options
    if (options.cdn !== undefined) {
      if (!_.isObject(options.cdn) ||
            !_.isString(options.cdn.url) ||
            (options.cdn.enabled !== undefined && !_.isBoolean(options.cdn.enabled))
      ) {
        return callback('CDN must be a valid object: {enabled: boolean, url: string}');
      }
      if (options.cdn.enabled === undefined) {
        options.cdn.enabled = true;
      }
      self.cdn = options.cdn;
    }

    // Load image backend
    self._image = options.image;
    if (typeof (self._image) === 'string') {
      self._image = require('./lib/image/' + self._image + '.js')();
    }

    // Reasonable default JPEG quality setting for scaled copies. Imagemagick's default
    // quality is the quality of the original being converted, which is usually a terrible idea
    // when it's a super hi res original. And if that isn't apropos it defaults
    // to 92 which is still sky high and produces very large files

    scaledJpegQuality = options.scaledJpegQuality || 80;

    imageSizes = options.imageSizes || [];

    async.series([
      // create temp folder if needed
      function (callback) {
        if (!imageSizes.length) {
          return callback();
        }

        tempPath = options.tempPath;

        if (!fs.existsSync(options.tempPath)) {
          fs.mkdirSync(options.tempPath);
        }
        return callback(null);
      },

      // invoke storage backend init with options
      function (callback) {
        return self._storage.init(options, callback);
      },

      // Autodetect image backend if necessary
      function (callback) {
        if (!self._image) {
          var paths = (process.env.PATH || '').split(delimiter);
          if (!_.find(paths, function(p) {
            if (fs.existsSync(p + '/imagecrunch')) {
              self._image = require('./lib/image/imagecrunch.js')();
              return true;
            }
            // Allow for Windows and Unix filenames for identify. Silly oversight
            // after getting delimiter right (:
            if (fs.existsSync(p + '/identify') || fs.existsSync(p + '/identify.exe')) {
              self._image = require('./lib/image/imagemagick.js')();
              return true;
            }
          })) {
            // Fall back to jimp, no need for an error
            self._image = require('./lib/image/jimp.js')();
          }
        }
        return callback(null);
      },

      // invoke image backend init with options
      function (callback) {
        return self._image.init(options, callback);
      }

    ], callback);
  };

  /**
   * The copyIn method takes a local filename and copies it to a path in uploadfs. Any intermediate folders that do not exist are automatically created if the storage requires such things. Just copy things where you want them to go.
   * @param  {[String]}   localPath   The local filename
   * @param  {[String]}   path    The path in uploadfs, begins with /
   * @param  {[Object]}   options    Options (passed to storage). May be skipped
   * @param  {Function} callback    Will receive the usual err argument
   */
  self.copyIn = function (localPath, path, options, callback) {
    if (typeof (options) === 'function') {
      callback = options;
      options = {};
    }
    path = prefixPath(path);
    return self._storage.copyIn(localPath, path, options, callback);
  };

  /**
   * Obtain the temporary folder used for intermediate files created by copyImageIn. Can also be useful when doing your own manipulations with copyOut.
   * @see Uploadfs#copyOut
   */
  self.getTempPath = function() {
    return tempPath;
  };

  /**
   * The copyOut method takes a path in uploadfs and a local filename and copies the file back from uploadfs to the local filesystem. This should be used only rarely. Heavy reliance on this method sets you up for poor performance in S3. However it may be necessary at times, for instance when you want to crop an image differently later. Use it only for occasional operations like cropping.
   * @param  {String}   path    Path in uploadfs (begins with /)
   * @param  {String}   localPath    Path in the local filesystem to copy to
   * @param  {Object}   options    Options (passed to backend). May be skipped
   * @param  {Function} callback    Receives the usual err argument
   */
  self.copyOut = function (path, localPath, options, callback) {
    path = prefixPath(path);
    if (typeof (options) === 'function') {
      callback = options;
      options = {};
    }
    return self._storage.copyOut(path, localPath, options, callback);
  };

  /**
   * Copy an image into uploadfs. Scaled versions as defined by the imageSizes option
   * at init() time are copied into uploadfs as follows:
   *
   * If 'path' is '/me' and sizes with names 'small', 'medium' and 'large'
   * were defined at init() time, the scaled versions will be:
   *
   * '/me.small.jpg', '/me.medium.jpg', '/me.large.jpg'
   *
   * And the original file will be copied to:
   *
   * '/me.jpg'
   *
   * Note that a file extension is added automatically. If you provide a
   * file extension in 'path' it will be honored when copying the original only.
   * The scaled versions will get appropriate extensions for their format
   * as detected by gm.
   *
   * If there is no error the second argument passed to the callback will
   * be an object with a 'basePath' property containing your original path
   * with the file extension removed and an 'extension' property containing
   * the automatically added file extension, as a convenience for locating the
   * original and scaled versions just by adding .jpg, .small.jpg, .medium.jpg,
   * etc.
   *
   * Scaled versions have the same file format as the original and are no wider
   * or taller than specified by the width and height properties of the
   * corresponding size, with the aspect ratio always being preserved.
   * If options.copyOriginal is explicitly false, the original image is
   * not copied into uploadfs at all.
   *
   * If options.crop is present, the image is cropped according to the
   * top, left, width and height properties of options.crop. All properties must be integers.
   * If cropping is done, it is performed first before scaling.
   *
   * IMPORTANT: if options.crop is present, the uncropped original is
   * NOT copied into uploadfs. The cropped version is what is copied
   * to "path." If you want the uncropped original too, make a separate call
   * to copyIn. A common pattern is to copy the original when an image
   * is first uploaded, and to perform crops and save them under other names
   * later, when a user decides they want cropped versions.
   *
   * Image scaling is performed with imagemagick, which must be installed
   * (note that Heroku provides it). In no case is an image ever scaled to
   * be larger than the original. Scaled versions of images with an orientation
   * hint, such as iPhone photographs, are automatically rotated by gm
   * so that they will display properly in web browsers.
   *
   * @param {String} localPath    Local filesystem path of existing image file
   * @param {String} path    Path in uploadfs to copy original to. Leave off the extension to autodetect the true type. Path begins with /
   * @param {Object} options Options: scaledJpegQuality, copyOriginal, crop (see above)
   * @param {Function} callback Receives the usual err argument
   */

  self.copyImageIn = function (localPath, path, options, callback) {

    // We do not call prefixPath here because we rely on copyIn, which does

    if (typeof (options) === 'function') {
      callback = options;
      options = {};
    }

    // We'll pass this context to the image processing backend with
    // additional properties
    var context = {
      crop: options.crop,
      sizes: imageSizes
    };

    context.scaledJpegQuality = options.scaledJpegQuality || scaledJpegQuality;

    // Identify the file type, size, etc. Stuff them into context.info and
    // context.extension

    function identify(path, callback) {
      return self.identifyLocalImage(path, function(err, info) {
        if (err) {
          return callback(err);
        }
        context.info = info;
        context.extension = info.extension;
        return callback(null);
      });
    }

    var originalDone = false;
    var copyOriginal = options.copyOriginal !== false;
    var originalPath;

    async.series({
      // Identify the file
      identify: function (callback) {
        return identify(localPath, function(err) {
          if (err) {
            return callback(err);
          }
          return callback(null);
        });
      },
      // make a temporary folder for our work
      temporary: function (callback) {
        // Name the destination folder
        context.tempName = generateId();
        // Create destination folder
        if (imageSizes.length) {
          context.tempFolder = tempPath + '/' + context.tempName;
          return fs.mkdir(context.tempFolder, callback);
        } else {
          return callback(null);
        }
      },
      // Determine base path in uploadfs, working path for temporary files,
      // and final uploadfs path of the original
      paths: function (callback) {
        context.basePath = path.replace(/\.\w+$/, '');
        context.workingPath = localPath;

        // Indulge their wild claims about the extension the original
        // should have if any, otherwise provide the truth from identify
        if (path.match(/\.\w+$/)) {
          originalPath = path;
        } else {
          originalPath = path + '.' + context.extension;
        }
        return callback(null);
      },
      copyOriginal: function(callback) {
        // If there are no transformations of the original, copy it
        // in directly
        if ((!copyOriginal) || (options.orientOriginals !== false) || (options.crop)) {
          return callback(null);
        }
        originalDone = true;
        return self.copyIn(localPath, originalPath, options, callback);
      },

      convert: function (callback) {
        context.copyOriginal = copyOriginal && (!originalDone);
        return async.series([
          convert,
          postprocess
        ], callback);
        function convert(callback) {
          return self._image.convert(context, callback);
        }
        function postprocess(callback) {
          if (!context.tempFolder) {
            // Nowhere to do the work
            return callback(null);
          }
          var filenames = _.map(imageSizes, function(size) {
            return context.tempFolder + '/' + size.name + '.' + context.extension;
          });
          return self.postprocess(filenames, callback);
        }
      },

      reidentify: function(callback) {
        if (!context.adjustedOriginal) {
          return callback(null);
        }
        // Push and pop the original size properties as we determined
        // those on the first identify and don't want to return the values
        // for the cropped and/or reoriented version
        var originalWidth = context.info.originalWidth;
        var originalHeight = context.info.originalHeight;
        return identify(context.adjustedOriginal, function(err) {
          if (err) {
            return callback(err);
          }
          context.info.originalWidth = originalWidth;
          context.info.originalHeight = originalHeight;
          return callback(null);
        });
      },

      copySizes: function(callback) {
        return async.each(imageSizes, function(size, callback) {
          var suffix = size.name + '.' + context.extension;
          var tempFile = context.tempFolder + '/' + suffix;
          var permFile = context.basePath + '.' + suffix;
          return self.copyIn(tempFile, permFile, options, callback);
        }, callback);
      },

      copyAdjustedOriginal: function(callback) {
        if (!context.adjustedOriginal) {
          return callback(null);
        }
        return self.copyIn(context.adjustedOriginal, originalPath, options, callback);
      }
    }, function (err) {
      // Try to clean up the temp folder. This can fail if its creation
      // failed, in which case there is nothing we can or should do,
      // thus the empty callback
      if (context.tempFolder) {
        rmRf(context.tempFolder, function (e) { });
      }
      callback(err, err ? null : {
        basePath: context.basePath,
        extension: context.extension,
        width: context.info.width,
        height: context.info.height,
        originalWidth: context.info.originalWidth,
        originalHeight: context.info.originalHeight
      });
    });
  };

  self.getUrl = function (options, callback) {
    if (self.cdn && self.cdn.enabled) {
      return self.cdn.url;
    }
    return self._storage.getUrl(options, callback) + self.prefix;
  };

  self.remove = function (path, callback) {
    path = prefixPath(path);
    return self._storage.remove(path, callback);
  };

  /**
   * Re-enable access to the file. By default newly uploaded
   * files ARE web accessible, so you need not call this method
   * unless uploadfs.disable has been previously called.
   *
   * Be aware that you MUST call this method to guarantee access
   * to the file via copyOut, as well as via the web, even though
   * some backends may only disable access via the web. Do not
   * rely on this behavior. (Differences in behavior between
   * local filesystems and S3 require we tolerate this difference.)
   *
   * @param  {string}   path     Path as stored in uploadfs (with extension)
   * @param  {Function} callback Receives error if any, otherwise null
   */

  self.enable = function (path, callback) {
    path = prefixPath(path);
    return self._storage.enable(path, callback);
  };

  /**
   * Disable web access to the file. By default new uploads ARE
   * accessible; however this method is useful when implementing a
   * "recycle bin" or other undo-able delete feature.
   *
   * The implementation MUST block web access to the file. The
   * implementation MAY also block read access via copyOut, so be
   * aware that you MUST call uploadfs.enable to reenable access to
   * the file to guarantee you have access to it again across all
   * storage backends, even if you are using copyOut to access it.
   *
   * @param  {string}   path     Path as stored in uploadfs (with extension)
   * @param  {Function} callback Receives error if any, otherwise null
   */

  self.disable = function (path, callback) {
    path = prefixPath(path);
    return self._storage.disable(path, callback);
  };

  /**
   * Identify a local image file. Normally you don't need to call
   * this yourself, it is mostly used by copyImageIn. But you may find it
   * useful in certain migration situations, so we have exported it.
   *
   * If the file is not an image or is too defective to be identified an error is
   * passed to the callback.
   *
   * Otherwise the second argument to the callback is guaranteed to have extension, width,
   * height, orientation, originalWidth and originalHeight properties. extension will be
   * gif, jpg or png and is detected from the file's true contents, not the original file
   * extension. width and height are automatically rotated to TopLeft orientation while
   * originalWidth and originalHeight are not.
   *
   * If the orientation property is not explicitly set in the file it will be set to
   * 'Undefined'.
   *
   * Alternative backends such as "sip" that do not support orientation detection
   * will not set this property at all.
   *
   * Any other properties returned are dependent on the version of ImageMagick (or
   * other backend) used and are not guaranteed.
   *
   * @param {String} path Local filesystem path to image file
   * @param {Function} callback Receives the usual err argument, followed by an object with extension, width, height, orientation, originalWidth and originalHeight properties. Any other properties depend on the backend in use and are not guaranteed
   *
   * @see Uploadfs#copyImageIn
   */

  self.identifyLocalImage = function(path, callback) {
    return self._image.identify(path, callback);
  };

  /**
   * Returns the image sizes array with which uploadfs was configured.
   * This may be of use if you must iterate over the various generated
   * images later.
   *
   * However note that a best practice is to retain information about the sizes
   * that were expected when each image was actually uploaded, because you might
   * change your mind and add or remove sizes later.
   * @return {array} [Image size objects]
   */
  self.getImageSizes = function() {
    return imageSizes;
  };

  /**
   * Destroys the uploadfs instance, allowing the backends to release any
   * resources they may be holding, such as file descriptors or interval timers.
   * Backends that hold such resources should implement their own `destroy` method,
   * also accepting a callback. The callback will receive an error if anything
   * goes awry during the cleanup process. This method does NOT remove any
   * content, it just releases system resources.
   * @param {function} callback
   */
  self.destroy = function(callback) {
    var callbacks = [
      self._storage.destroy || noOperation,
      self._image.destroy || noOperation
    ];
    return async.parallel(callbacks, callback);
    function noOperation(callback) {
      return callback(null);
    }
  };

  self.migrateToDisabledFileKey = function(callback) {
    var method = self._storage.migrateToDisabledFileKey;
    if (!method) {
      // Not relevant for this backend
      return callback(null);
    }
    return self._storage.migrateToDisabledFileKey(callback);
  };

  self.migrateFromDisabledFileKey = function(callback) {
    var method = self._storage.migrateFromDisabledFileKey;
    if (!method) {
      // Not relevant for this backend
      return callback(null);
    }
    return self._storage.migrateFromDisabledFileKey(callback);
  };

  // Called by `convert` to postprocess resized/cropped images
  // for optimal file size, etc.

  self.postprocess = function(files, callback) {
    var sample = files[0];
    if (!sample) {
      return callback(null);
    }
    var relevant = _.filter(self.options.postprocessors || [], function(postprocessor) {
      var matches = sample.match(/\.(\w+)$/);
      if (!matches) {
        return false;
      }
      var extension = matches[1];
      return _.includes(postprocessor.extensions, extension);
    });
    var folder = require('path').dirname(sample);
    return async.eachSeries(relevant, function(postprocessor, callback) {
      if (postprocessor.length === 4) {
        return postprocessor.postprocessor(files, folder, postprocessor.options, callback);
      } else {
        return postprocessor.postprocessor(files, folder, postprocessor.options).then(function() {
          return callback(null);
        }).catch(function(err) {
          return callback(err);
        });
      }
    }, callback);
  };

  function prefixPath(path) {
    // Resolve any double // that results from the prefix
    return (self.prefix + path).replace(/\/\//g, '/');
  }

}
845541
def _process_upload_descriptor(self, ud):
        # type: (Uploader, blobxfer.models.upload.Descriptor) -> None
        
        # get upload offsets
        offsets, resume_bytes = ud.next_offsets()
        # add resume bytes to counter
        if resume_bytes is not None:
            with self._transfer_lock:
                self._upload_bytes_total += ud.entity.size
                self._upload_bytes_sofar += resume_bytes
                logger.debug('adding {} sofar {} from {}'.format(
                    resume_bytes, self._upload_bytes_sofar, ud.entity.name))
            del resume_bytes
        # check if all operations completed
        if offsets is None and ud.all_operations_completed:
            # finalize file
            self._finalize_upload(ud)
            # accounting
            with self._upload_lock:
                if ud.entity.is_encrypted:
                    self._ud_map.pop(ud.unique_id)
                self._upload_set.remove(ud.unique_id)
                self._upload_sofar += 1
            return
        # if nothing to upload, re-enqueue for finalization
        if offsets is None:
            self._upload_queue.put(ud)
            return
        # prepare upload
        if offsets.chunk_num == 0:
            self._prepare_upload(ud.entity)
        # prepare replicae targets
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                if offsets.chunk_num == 0:
                    self._prepare_upload(ase)
        # encrypt if necessary
        if ud.entity.is_encrypted and ud.entity.size > 0:
            # send iv through hmac if first chunk
            if offsets.chunk_num == 0:
                ud.hmac_data(ud.current_iv)
            # encrypt data
            if self._crypto_offload is None:
                # read data from file and encrypt
                data, _ = ud.read_data(offsets)
                encdata = blobxfer.operations.crypto.aes_cbc_encrypt_data(
                    ud.entity.encryption_metadata.symmetric_key,
                    ud.current_iv, data, offsets.pad)
                # send encrypted data through hmac
                ud.hmac_data(encdata)
                data = encdata
                # save last 16 encrypted bytes for next IV
                ud.current_iv = \
                    encdata[-blobxfer.models.crypto.AES256_BLOCKSIZE_BYTES:]
            else:  # noqa
                # crypto offload is not supported with AES256-CBC FullBlob
                raise NotImplementedError()
                # self._crypto_offload.add_encrypt_chunk(
                #     str(ud.local_path.absolute_path), offsets,
                #     ud.entity.encryption_metadata.symmetric_key,
                #     ud.current_iv)
                # encrypted data will be retrieved from a temp file once
                # retrieved from crypto queue
                # return_early = True
        else:
            data, newoffset = ud.read_data(offsets)
            # set new offset if stdin
            if newoffset is not None:
                offsets = newoffset
        # re-enqueue for other threads to upload if not append
        if ud.entity.mode != blobxfer.models.azure.StorageModes.Append:
            self._upload_queue.put(ud)
        # no data can be returned on stdin uploads
        if ud.local_path.use_stdin and not data:
            return
        # add data to transfer queue
        with self._transfer_lock:
            self._transfer_set.add(
                blobxfer.operations.upload.Uploader.create_unique_transfer_id(
                    ud.local_path, ud.entity, offsets))
        self._transfer_queue.put((ud, ud.entity, offsets, data))
        # iterate replicas
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                with self._transfer_lock:
                    self._transfer_set.add(
                        blobxfer.operations.upload.Uploader.
                        create_unique_transfer_id(ud.local_path, ase, offsets)
                    )
                self._transfer_queue.put((ud, ase, offsets, data))
1132625
private <T extends IEntity> IntuitMessage prepareUpload(T entity, InputStream docContent, String boundaryId) throws FMSException {
        IntuitMessage intuitMessage = new IntuitMessage();
        RequestElements requestElements = intuitMessage.getRequestElements();

        //set the request parameters
        Map<String, String> requestParameters = requestElements.getRequestParameters();
        requestParameters.put(RequestElements.REQ_PARAM_METHOD_TYPE, MethodType.POST.toString());

        UploadRequestElements uploadRequestElements = requestElements.getUploadRequestElements();
        uploadRequestElements.setDocContent(docContent);
        uploadRequestElements.setBoundaryId( utilizeIf(boundaryId) );
        uploadRequestElements.setElementsId(generateId().substring(0, 5));

        requestElements.setAction(OperationType.UPLOAD.toString());
        requestElements.setContext(context);
        requestElements.setEntity(entity);
        requestElements.setObjectToSerialize(getSerializableObject(entity));

        return intuitMessage;
    }
867793
def _finalize_upload(self, sd):
        # type: (SyncCopy, blobxfer.models.synccopy.Descriptor) -> None
        
        metadata = sd.src_entity.raw_metadata
        if blobxfer.util.is_not_empty(sd.src_entity.md5):
            digest = sd.src_entity.md5
        else:
            digest = None
        if sd.requires_put_block_list:
            # put block list for non one-shot block blobs
            self._finalize_block_blob(sd, metadata, digest)
        elif sd.remote_is_page_blob or sd.remote_is_append_blob:
            # append and page blob finalization
            self._finalize_nonblock_blob(sd, metadata, digest)
        elif sd.remote_is_file:
            # azure file finalization
            self._finalize_azure_file(sd, metadata, digest)
        # set access tier
        if sd.requires_access_tier_set:
            blobxfer.operations.azure.blob.block.set_blob_access_tier(
                sd.dst_entity)