Bind source storage entity to destination storage entities
        :param SyncCopy self: this
        :rtype: tuple
        :return: (source storage entity, destination storage entity)
761652
def _bind_sources_to_destination(self):
        # type: (SyncCopy) ->
        #        Tuple[blobxfer.models.azure.StorageEntity,
        #        blobxfer.models.azure.StorageEntity]
        
        seen = set()
        # iterate through source paths to download
        for src in self._spec.sources:
            for src_ase in src.files(
                    self._creds, self._spec.options,
                    self._general_options.dry_run):
                # generate copy destinations for source
                dest = [
                    dst_ase for dst_ase in
                    self._generate_destination_for_source(src_ase)
                ]
                if len(dest) == 0:
                    continue
                primary_dst = dest[0]
                uid = blobxfer.operations.synccopy.SyncCopy.create_deletion_id(
                    primary_dst._client, primary_dst.container,
                    primary_dst.name)
                if uid in seen:
                    raise RuntimeError(
                        'duplicate destination entity detected: {}/{}'.format(
                            primary_dst._client.primary_endpoint,
                            primary_dst.path))
                seen.add(uid)
                # add to exclusion set
                if self._spec.options.delete_extraneous_destination:
                    self._delete_exclude.add(uid)
                if len(dest[1:]) > 0:
                    if primary_dst.replica_targets is None:
                        primary_dst.replica_targets = []
                    primary_dst.replica_targets.extend(dest[1:])
                    # check replica targets for duplicates
                    for rt in primary_dst.replica_targets:
                        ruid = (
                            blobxfer.operations.synccopy.SyncCopy.
                            create_deletion_id(
                                rt._client, rt.container, rt.name)
                        )
                        if ruid in seen:
                            raise RuntimeError(
                                ('duplicate destination entity detected: '
                                 '{}/{}').format(
                                     rt._client.primary_endpoint, rt.path))
                        seen.add(ruid)
                        # add replica targets to deletion exclusion set
                        if self._spec.options.delete_extraneous_destination:
                            self._delete_exclude.add(ruid)
                yield src_ase, primary_dst
819188
def merged_sourcekey(self, **kwargs):
        
        kwargs_copy = self.base_dict.copy()
        kwargs_copy.update(**kwargs)
        self._replace_none(kwargs_copy)        
        try:
            return NameFactory.merged_sourcekey_format.format(**kwargs_copy)
        except KeyError:
            return None
656053
def copy(source, destination):
    
    if os.path.isdir(source):
        return __copytree(source, destination)
    else:
        return __copyfile2(source, destination)
722408
def get(self, key):
        
        return SyncMapItemContext(
            self._version,
            service_sid=self._solution['service_sid'],
            map_sid=self._solution['map_sid'],
            key=key,
        )
1168506
def _sync_download(self, url, destination_path):
    """"""
    proxies = {
        'http': os.environ.get('TFDS_HTTP_PROXY', None),
        'https': os.environ.get('TFDS_HTTPS_PROXY', None),
        'ftp': os.environ.get('TFDS_FTP_PROXY', None)
    }
    if kaggle.KaggleFile.is_kaggle_url(url):
      if proxies['http']:
        os.environ['KAGGLE_PROXY'] = proxies['http']
      return self._sync_kaggle_download(url, destination_path)

    try:
      # If url is on a filesystem that gfile understands, use copy. Otherwise,
      # use requests.
      if not url.startswith('http'):
        return self._sync_file_copy(url, destination_path)
    except tf.errors.UnimplementedError:
      pass

    session = requests.Session()
    session.proxies = proxies
    if _DRIVE_URL.match(url):
      url = self._get_drive_url(url, session)
    use_urllib = url.startswith('ftp')
    if use_urllib:
      if proxies['ftp']:
        proxy = urllib.request.ProxyHandler({'ftp': proxies['ftp']})
        opener = urllib.request.build_opener(proxy)
        urllib.request.install_opener(opener)   # pylint: disable=too-many-function-args
      request = urllib.request.Request(url)
      response = urllib.request.urlopen(request)
    else:
      response = session.get(url, stream=True)
      if response.status_code != 200:
        raise DownloadError('Failed to get url %s. HTTP code: %d.' %
                            (url, response.status_code))
    fname = _get_filename(response)
    path = os.path.join(destination_path, fname)
    size = 0

    size_mb = 0
    unit_mb = units.MiB
    self._pbar_dl_size.update_total(
        int(response.headers.get('Content-length', 0)) // unit_mb)
    with tf.io.gfile.GFile(path, 'wb') as file_:
      checksum = self._checksumer()
      if use_urllib:
        iterator = iter(lambda: response.read(io.DEFAULT_BUFFER_SIZE), b'')
      else:
        iterator = response.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)

      for block in iterator:
        size += len(block)

        # Update the progress bar
        size_mb += len(block)
        if size_mb > unit_mb:
          self._pbar_dl_size.update(size_mb // unit_mb)
          size_mb %= unit_mb

        checksum.update(block)
        file_.write(block)
    self._pbar_url.update(1)
    return checksum.hexdigest(), size
1080025
def synccopy(ctx):
    """"""
    settings.add_cli_options(ctx.cli_options, settings.TransferAction.Synccopy)
    ctx.initialize(settings.TransferAction.Synccopy)
    specs = settings.create_synccopy_specifications(
        ctx.cli_options, ctx.config)
    del ctx.cli_options
    for spec in specs:
        blobxfer.api.SyncCopy(
            ctx.general_options, ctx.credentials, spec
        ).start()
1186538
def sync_in(self, force=False):
        """"""
        self.log('---- Sync In ----')

        self.dstate = self.STATES.BUILDING

        for path_name in self.source_fs.listdir():

            f = self.build_source_files.instance_from_name(path_name)

            if not f:
                self.warn('Ignoring unknown file: {}'.format(path_name))
                continue

            if f and f.exists and (f.fs_is_newer or force):
                self.log('Sync: {}'.format(f.record.path))
                f.fs_to_record()
                f.record_to_objects()

        self.commit()

        self.library.search.index_bundle(self, force=True)
496944
public static void syncFileRangeIfPossible(
      FileDescriptor fd, long offset, long nbytes, int flags)
      throws NativeIOException {
    InjectionHandler.processEvent(InjectionEventCore.NATIVEIO_SYNC_FILE_RANGE,
        flags);
    if (nativeLoaded && syncFileRangePossible) {
      try {
        sync_file_range(fd, offset, nbytes, flags);
      } catch (UnsupportedOperationException uoe) {
        LOG.warn("syncFileRangeIfPossible() failed", uoe);
        syncFileRangePossible = false;
      } catch (UnsatisfiedLinkError ule) {
        LOG.warn("syncFileRangeIfPossible() failed", ule);
        syncFileRangePossible = false;
      } catch (NativeIOException nie) {
        LOG.warn("syncFileRangeIfPossible() failed: fd " + fd + " offset "
            + offset + " nbytes " + nbytes + " flags " + flags, nie);
        throw nie;
      }
    }
  }
666737
def bind(self, instance):
        

        p = self.clone()
        p.instance = weakref.ref(instance)
        return p
821195
def sync_objects_out(self, force=False):
        """"""
        self.log('---- Sync Objects Out ----')
        from ambry.bundle.files import BuildSourceFile

        self.dstate = self.STATES.BUILDING

        for f in self.build_source_files.list_records():

            self.log('Sync: {}'.format(f.record.path))
            f.objects_to_record()

        self.commit()
609055
def sync(self, json_obj=None):
        
        LOGGER.debug("Transport.sync")
        if json_obj is None:
            params = None
            if self.id is not None:
                params = SessionService.complete_transactional_req({'ID': self.id})

            if params is not None:
                if MappingService.driver_type != DriverFactory.DRIVER_REST:
                    params['OPERATION'] = 'getTransport'
                    args = {'properties': params}
                else:
                    args = {'http_operation': 'GET', 'operation_path': 'get', 'parameters': params}

                response = TransportService.requester.call(args)

                if MappingService.driver_type != DriverFactory.DRIVER_REST:
                    response = response.get()

                if response.rc == 0:
                    json_obj = response.response_content
                else:
                    err_msg = 'Transport.sync - Problem while syncing transport (id: ' + str(self.id) + '). ' \
                              'Reason: ' + str(response.response_content) + ' - ' + str(response.error_message) + \
                              " (" + str(response.rc) + ")"
                    LOGGER.warning(err_msg)
                    if response.rc == 500 and ArianeMappingOverloadError.ERROR_MSG in response.error_message:
                        raise ArianeMappingOverloadError("Transport.sync",
                                                         ArianeMappingOverloadError.ERROR_MSG)
                    # traceback.print_stack()
        elif 'transportID' not in json_obj:
            err_msg = 'Transport.sync - Problem while syncing transport (id: ' + str(self.id) + '). ' \
                      'Reason: inconsistent json_obj' + str(json_obj) + " from : \n"
            LOGGER.warning(err_msg)
            # traceback.print_stack()

        if json_obj is not None:
            self.id = json_obj['transportID']
            self.name = json_obj['transportName']
            if MappingService.driver_type != DriverFactory.DRIVER_REST:
                if 'transportProperties' in json_obj:
                    self.properties = DriverTools.json2properties(json_obj['transportProperties'])
                else:
                    self.properties = None
            else:
                self.properties = json_obj['transportProperties'] if 'transportProperties' in json_obj else None
542974
def Copier(source, destination):
    

    if source.type == 'local' and destination.type == 'local':
        return LocalCopier(source, destination)
    elif source.type == 'local' and destination.type == 'google_storage':
        return Local2GoogleStorageCopier(source, destination)
    elif source.type == 'google_storage' and destination.type == 'local':
        return GoogleStorage2LocalCopier(source, destination)
    elif source.type == 'google_storage' and destination.type == 'google_storage':
        return GoogleStorageCopier(source, destination)
    else:
        raise FileUtilsError('Could not find method to copy from source '\
                        '"%s" to destination "%s".' % (source, destination))
92293
def sync(self):
        """"""
        if self.filepath:
            serialised_file = open(self.filepath, "w")
            json.dump(self.state, serialised_file)
            serialised_file.close()
        else:
            print("Filepath to the persistence file is not set. State cannot be synced to disc.")
1198712
def bind_to_storage_buffer(self, binding=0, *, offset=0, size=-1) -> None:
        '''
            
        '''

        self.mglo.bind_to_storage_buffer(binding, offset, size)
380521
def merge(self, storagemodel) -> StorageTableModel:
        
        modeldefinition = self.getmodeldefinition(storagemodel, True)
       
        try:
            pk = storagemodel.getPartitionKey()
            rk = storagemodel.getRowKey()
            entity = modeldefinition['tableservice'].get_entity(modeldefinition['tablename'], pk, rk)
        
            """ merge with entity values """
            for key, default in vars(storagemodel.__class__).items():
                if not key.startswith('_') and key not in ['']:

                    if isinstance(default, PartitionKey) or isinstance(default, RowKey) or  isinstance(default, EncryptKey):
                        default = default._default

                    newvalue = getattr(storagemodel, key, None)
                    if (newvalue is None) or (newvalue == default):
                        oldvalue = getattr(entity, key, default)
                        setattr(storagemodel, key, oldvalue)
            
            modeldefinition['tableservice'].insert_or_replace_entity(modeldefinition['tablename'], storagemodel.entity())
            storagemodel._exists = True

        except AzureMissingResourceHttpError as e:
            log.debug('can not merge table entity:  Table {}, PartitionKey {}, RowKey {} because {!s}'.format(modeldefinition['tablename'], pk, rk, e))

        except Exception as e:
            log.debug('can not merge table entity:  Table {}, PartitionKey {}, RowKey {} because {!s}'.format(modeldefinition['tablename'], pk, rk, e))

        finally:
            return storagemodel
1303288
def sync(client=None, force=False, verbose=True):
    
    from mediasync import backends
    from mediasync.conf import msettings
    from mediasync.signals import pre_sync, post_sync

    # create client connection
    if client is None:
        client = backends.client()

    client.open()
    client.serve_remote = True

    # send pre-sync signal
    pre_sync.send(sender=client)

    #
    # sync joined media
    #

    for joinfile, sourcefiles in msettings['JOINED'].iteritems():
        
        filedata = combine_files(joinfile, sourcefiles, client)
        if filedata is None:
            # combine_files() is only interested in CSS/JS files.
            continue
        filedata, dirname = filedata

        content_type = mimetypes.guess_type(joinfile)[0] or msettings['DEFAULT_MIMETYPE']

        remote_path = joinfile
        if dirname:
            remote_path = "%s/%s" % (dirname, remote_path)

        if client.process_and_put(filedata, content_type, remote_path, force=force):
            if verbose:
                print "[%s] %s" % (content_type, remote_path)

    #
    # sync static media
    #

    for dirname in os.listdir(client.media_root):

        dirpath = os.path.abspath(os.path.join(client.media_root, dirname))

        if os.path.isdir(dirpath):

            for filename in listdir_recursive(dirpath):

                # calculate local and remote paths
                filepath = os.path.join(dirpath, filename)
                remote_path = "%s/%s" % (dirname, filename)

                content_type = mimetypes.guess_type(filepath)[0] or msettings['DEFAULT_MIMETYPE']

                if not is_syncable_file(os.path.basename(filename)) or not os.path.isfile(filepath):
                    continue # hidden file or directory, do not upload

                filedata = open(filepath, 'rb').read()

                if client.process_and_put(filedata, content_type, remote_path, force=force):
                    if verbose:
                        print "[%s] %s" % (content_type, remote_path)
    
    # send post-sync signal while client is still open
    post_sync.send(sender=client)
    
    client.close()
45717
def remote_is_append_blob(self):
        # type: (Descriptor) -> bool
        
        return self.entity.mode == blobxfer.models.azure.StorageModes.Append