Get a single file from the salt-master
        path must be a salt server location, aka, salt://path/to/file, if
        dest is omitted, then the downloaded file will be placed in the minion
        cache
774495
def get_file(self,
                 path,
                 dest='',
                 makedirs=False,
                 saltenv='base',
                 gzip=None,
                 cachedir=None):
        '''
        
        '''
        path, senv = salt.utils.url.split_env(path)
        if senv:
            saltenv = senv

        if not salt.utils.platform.is_windows():
            hash_server, stat_server = self.hash_and_stat_file(path, saltenv)
            try:
                mode_server = stat_server[0]
            except (IndexError, TypeError):
                mode_server = None
        else:
            hash_server = self.hash_file(path, saltenv)
            mode_server = None

        # Check if file exists on server, before creating files and
        # directories
        if hash_server == '':
            log.debug(
                'Could not find file \'%s\' in saltenv \'%s\'',
                path, saltenv
            )
            return False

        # If dest is a directory, rewrite dest with filename
        if dest is not None \
                and (os.path.isdir(dest) or dest.endswith(('/', '\\'))):
            dest = os.path.join(dest, os.path.basename(path))
            log.debug(
                'In saltenv \'%s\', \'%s\' is a directory. Changing dest to '
                '\'%s\'', saltenv, os.path.dirname(dest), dest
            )

        # Hash compare local copy with master and skip download
        # if no difference found.
        dest2check = dest
        if not dest2check:
            rel_path = self._check_proto(path)

            log.debug(
                'In saltenv \'%s\', looking at rel_path \'%s\' to resolve '
                '\'%s\'', saltenv, rel_path, path
            )
            with self._cache_loc(
                    rel_path, saltenv, cachedir=cachedir) as cache_dest:
                dest2check = cache_dest

        log.debug(
            'In saltenv \'%s\', ** considering ** path \'%s\' to resolve '
            '\'%s\'', saltenv, dest2check, path
        )

        if dest2check and os.path.isfile(dest2check):
            if not salt.utils.platform.is_windows():
                hash_local, stat_local = \
                    self.hash_and_stat_file(dest2check, saltenv)
                try:
                    mode_local = stat_local[0]
                except (IndexError, TypeError):
                    mode_local = None
            else:
                hash_local = self.hash_file(dest2check, saltenv)
                mode_local = None

            if hash_local == hash_server:
                return dest2check

        log.debug(
            'Fetching file from saltenv \'%s\', ** attempting ** \'%s\'',
            saltenv, path
        )
        d_tries = 0
        transport_tries = 0
        path = self._check_proto(path)
        load = {'path': path,
                'saltenv': saltenv,
                'cmd': '_serve_file'}
        if gzip:
            gzip = int(gzip)
            load['gzip'] = gzip

        fn_ = None
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    try:
                        os.makedirs(destdir)
                    except OSError as exc:
                        if exc.errno != errno.EEXIST:  # ignore if it was there already
                            raise
                else:
                    return False
            # We need an open filehandle here, that's why we're not using a
            # with clause:
            fn_ = salt.utils.files.fopen(dest, 'wb+')  # pylint: disable=resource-leakage
        else:
            log.debug('No dest file found')

        while True:
            if not fn_:
                load['loc'] = 0
            else:
                load['loc'] = fn_.tell()
            data = self.channel.send(load, raw=True)
            if six.PY3:
                # Sometimes the source is local (eg when using
                # 'salt.fileserver.FSChan'), in which case the keys are
                # already strings. Sometimes the source is remote, in which
                # case the keys are bytes due to raw mode. Standardize on
                # strings for the top-level keys to simplify things.
                data = decode_dict_keys_to_str(data)
            try:
                if not data['data']:
                    if not fn_ and data['dest']:
                        # This is a 0 byte file on the master
                        with self._cache_loc(
                                data['dest'],
                                saltenv,
                                cachedir=cachedir) as cache_dest:
                            dest = cache_dest
                            with salt.utils.files.fopen(cache_dest, 'wb+') as ofile:
                                ofile.write(data['data'])
                    if 'hsum' in data and d_tries < 3:
                        # Master has prompted a file verification, if the
                        # verification fails, re-download the file. Try 3 times
                        d_tries += 1
                        hsum = salt.utils.hashutils.get_hash(dest, salt.utils.stringutils.to_str(data.get('hash_type', b'md5')))
                        if hsum != data['hsum']:
                            log.warning(
                                'Bad download of file %s, attempt %d of 3',
                                path, d_tries
                            )
                            continue
                    break
                if not fn_:
                    with self._cache_loc(
                            data['dest'],
                            saltenv,
                            cachedir=cachedir) as cache_dest:
                        dest = cache_dest
                        # If a directory was formerly cached at this path, then
                        # remove it to avoid a traceback trying to write the file
                        if os.path.isdir(dest):
                            salt.utils.files.rm_rf(dest)
                        fn_ = salt.utils.atomicfile.atomic_open(dest, 'wb+')
                if data.get('gzip', None):
                    data = salt.utils.gzip_util.uncompress(data['data'])
                else:
                    data = data['data']
                if six.PY3 and isinstance(data, str):
                    data = data.encode()
                fn_.write(data)
            except (TypeError, KeyError) as exc:
                try:
                    data_type = type(data).__name__
                except AttributeError:
                    # Shouldn't happen, but don't let this cause a traceback.
                    data_type = six.text_type(type(data))
                transport_tries += 1
                log.warning(
                    'Data transport is broken, got: %s, type: %s, '
                    'exception: %s, attempt %d of 3',
                    data, data_type, exc, transport_tries
                )
                self._refresh_channel()
                if transport_tries > 3:
                    log.error(
                        'Data transport is broken, got: %s, type: %s, '
                        'exception: %s, retry attempts exhausted',
                        data, data_type, exc
                    )
                    break

        if fn_:
            fn_.close()
            log.info(
                'Fetching file from saltenv \'%s\', ** done ** \'%s\'',
                saltenv, path
            )
        else:
            log.debug(
                'In saltenv \'%s\', we are ** missing ** the file \'%s\'',
                saltenv, path
            )

        return dest
648689
def local_path(self, url, filename=None, decompress=False, download=False):
        
        if download:
            return self.fetch(url=url, filename=filename, decompress=decompress)
        else:
            filename = self.local_filename(url, filename, decompress)
            return join(self.cache_directory_path, filename)
915679
def cache_file(self, template):
        '''
        
        '''
        saltpath = salt.utils.url.create(template)
        self.file_client().get_file(saltpath, '', True, self.saltenv)
242268
def download(self, url, path, force_refetch=False, nocache=False):
        ''' 
        '''
        # Enable cache
        if os.path.isfile(path):
            getLogger().info("File exists, download task skipped -> {path}".format(path=path))
            return True
        try:
            # Open URL
            getLogger().info("Downloading: {url} -> {path}".format(url=url, path=path))
            response = self.fetch(url, force_refetch=force_refetch, nocache=nocache)
            if response is not None:
                # Download file
                local_file = open(path, "wb")
                local_file.write(response)
                local_file.close()
                # Finished
                return True
            else:
                return False
        except Exception as e:
            if hasattr(e, 'reason'):
                getLogger().exception('We failed to reach a server. Reason: %s' % (e.reason,))
            elif hasattr(e, 'code'):
                getLogger().exception("The server couldn't fulfill the request. Error code: {code}".format(code=e.code))
            else:
                # everything is fine
                getLogger().exception("Unknown error: %s" % (e,))
        return False
365200
def get(self, s3_path, destination_local_path):
        
        (bucket, key) = self._path_to_bucket_and_key(s3_path)
        # download the file
        self.s3.meta.client.download_file(bucket, key, destination_local_path)
1199530
def _extrn_path(self, url, saltenv, cachedir=None):
        '''
        
        '''
        url_data = urlparse(url)
        if salt.utils.platform.is_windows():
            netloc = salt.utils.path.sanitize_win_path(url_data.netloc)
        else:
            netloc = url_data.netloc

        # Strip user:pass from URLs
        netloc = netloc.split('@')[-1]

        if cachedir is None:
            cachedir = self.opts['cachedir']
        elif not os.path.isabs(cachedir):
            cachedir = os.path.join(self.opts['cachedir'], cachedir)

        if url_data.query:
            file_name = '-'.join([url_data.path, url_data.query])
        else:
            file_name = url_data.path

        if len(file_name) > MAX_FILENAME_LENGTH:
            file_name = salt.utils.hashutils.sha256_digest(file_name)

        return salt.utils.path.join(
            cachedir,
            'extrn_files',
            saltenv,
            netloc,
            file_name
        )
1284483
def _download_if_necessary(self, url, download_if_missing, overwrite):
        
        cached_path = self.cached_path(url)
        missing = not exists(cached_path)
        if (missing or overwrite) and download_if_missing:
            logger.info("Fetching %s from URL %s", cached_path, url)
            datacache.download._download_and_decompress_if_necessary(
                full_path=cached_path,
                download_url=url,
                timeout=3600)
        elif missing:
            raise MissingRemoteFile(url)
        return cached_path
53634
def cache_file(self, local_path, ip, path):
        
        destination = self.__destination(ip, path)
        atomicish_move(local_path, destination)
1281149
def recv(files, dest):
    '''
    
    '''
    ret = {}
    for path, data in six.iteritems(files):
        if os.path.basename(path) == os.path.basename(dest) \
                and not os.path.isdir(dest):
            final = dest
        elif os.path.isdir(dest):
            final = os.path.join(dest, os.path.basename(path))
        elif os.path.isdir(os.path.dirname(dest)):
            final = dest
        else:
            return 'Destination unavailable'

        try:
            with salt.utils.files.fopen(final, 'w+') as fp_:
                fp_.write(data)
            ret[final] = True
        except IOError:
            ret[final] = False

    return ret
637618
def _get_path(self, file):
        """"""
        dir = self._cache_directory()
        if not os.path.exists(dir):
            os.makedirs(dir)
        return os.path.join(dir, file)
67343
def fetch(bank, key, cachedir):
    '''
    
    '''
    inkey = False
    key_file = os.path.join(cachedir, os.path.normpath(bank), '{0}.p'.format(key))
    if not os.path.isfile(key_file):
        # The bank includes the full filename, and the key is inside the file
        key_file = os.path.join(cachedir, os.path.normpath(bank) + '.p')
        inkey = True

    if not os.path.isfile(key_file):
        log.debug('Cache file "%s" does not exist', key_file)
        return {}
    try:
        with salt.utils.files.fopen(key_file, 'rb') as fh_:
            if inkey:
                return __context__['serial'].load(fh_)[key]
            else:
                return __context__['serial'].load(fh_)
    except IOError as exc:
        raise SaltCacheError(
            'There was an error reading the cache file "{0}": {1}'.format(
                key_file, exc
            )
        )
605987
function downloadWithCache(url, dir, name) {
  const filePath = path.resolve(dir, name);
  const md5Name = name + '-sha1.txt';
  const mdFile = path.resolve(dir, md5Name);

  var downloadAndWriteSha1 = function(url, toBeHashed) {
    logger.info(`start to download: ${url}`);

    const result = requestSync('GET', url);

    fs.writeFileSync(filePath, result.getBody());
    fs.writeFileSync(mdFile, toBeHashed, {
      encoding: 'utf8',
      flag: 'w'
    });
    logger.info(`download success: ${filePath}`);
    return filePath;
  }

  if (_.isExistedFile(mdFile)) {
    const data = fs.readFileSync(mdFile, {
      encoding: 'utf8',
      flag: 'r'
    });
    logger.info(`get ${filePath} from cache`);
    logger.info(`sha:${data.trim()}`);
    return filePath;
  } else {
    const result = crypto.createHash('md5').update(url).digest('hex');
    try {
      return downloadAndWriteSha1(url, result);
    } catch (e) {
      console.log(e);
    }
  }
}
83879
def _cache_loc(self, path, saltenv='base', cachedir=None):
        '''
        
        '''
        cachedir = self.get_cachedir(cachedir)
        dest = salt.utils.path.join(cachedir,
                                    'files',
                                    saltenv,
                                    path)
        destdir = os.path.dirname(dest)
        with salt.utils.files.set_umask(0o077):
            # remove destdir if it is a regular file to avoid an OSError when
            # running os.makedirs below
            if os.path.isfile(destdir):
                os.remove(destdir)

            # ensure destdir exists
            try:
                os.makedirs(destdir)
            except OSError as exc:
                if exc.errno != errno.EEXIST:  # ignore if it was there already
                    raise

            yield dest
1332859
def __get_file_path(self, path, saltenv='base'):
        '''
        
        '''
        try:
            path = self._check_proto(path)
        except MinionError as err:
            # Local file path
            if not os.path.isfile(path):
                log.warning(
                    'specified file %s is not present to generate hash: %s',
                    path, err
                )
                return None
            else:
                return path
        return self._find_file(path, saltenv)
790936
def _file_recv(self, load):
        '''
        
        '''
        if any(key not in load for key in ('id', 'path', 'loc')):
            return False
        if not isinstance(load['path'], list):
            return False
        if not self.opts['file_recv']:
            return False
        if not salt.utils.verify.valid_id(self.opts, load['id']):
            return False
        if 'loc' in load and load['loc'] < 0:
            log.error('Invalid file pointer: load[loc] < 0')
            return False

        if len(load['data']) + load.get('loc', 0) > self.opts['file_recv_max_size'] * 0x100000:
            log.error(
                'file_recv_max_size limit of %d MB exceeded! %s will be '
                'truncated. To successfully push this file, adjust '
                'file_recv_max_size to an integer (in MB) large enough to '
                'accommodate it.', self.opts['file_recv_max_size'], load['path']
            )
            return False
        if 'tok' not in load:
            log.error(
                'Received incomplete call from %s for \'%s\', missing \'%s\'',
                load['id'], inspect_stack()['co_name'], 'tok'
            )
            return False
        if not self.__verify_minion(load['id'], load['tok']):
            # The minion is not who it says it is!
            # We don't want to listen to it!
            log.warning('Minion id %s is not who it says it is!', load['id'])
            return {}
        load.pop('tok')

        # Join path
        sep_path = os.sep.join(load['path'])

        # Path normalization should have been done by the sending
        # minion but we can't guarantee it. Re-do it here.
        normpath = os.path.normpath(sep_path)

        # Ensure that this safety check is done after the path
        # have been normalized.
        if os.path.isabs(normpath) or '../' in load['path']:
            # Can overwrite master files!!
            return False

        cpath = os.path.join(
            self.opts['cachedir'],
            'minions',
            load['id'],
            'files',
            normpath)
        # One last safety check here
        if not os.path.normpath(cpath).startswith(self.opts['cachedir']):
            log.warning(
                'Attempt to write received file outside of master cache '
                'directory! Requested path: %s. Access denied.', cpath
            )
            return False
        cdir = os.path.dirname(cpath)
        if not os.path.isdir(cdir):
            try:
                os.makedirs(cdir)
            except os.error:
                pass
        if os.path.isfile(cpath) and load['loc'] != 0:
            mode = 'ab'
        else:
            mode = 'wb'
        with salt.utils.files.fopen(cpath, mode) as fp_:
            if load['loc']:
                fp_.seek(load['loc'])

            fp_.write(salt.utils.stringutils.to_bytes(load['data']))
        return True
864069
def hash_and_stat_file(self, path, saltenv='base'):
        '''
        
        '''
        hash_result = self.hash_file(path, saltenv)
        try:
            path = self._check_proto(path)
        except MinionError as err:
            if not os.path.isfile(path):
                return hash_result, None
            else:
                try:
                    return hash_result, list(os.stat(path))
                except Exception:
                    return hash_result, None
        load = {'path': path,
                'saltenv': saltenv,
                'cmd': '_file_find'}
        fnd = self.channel.send(load)
        try:
            stat_result = fnd.get('stat')
        except AttributeError:
            stat_result = None
        return hash_result, stat_result
1010711
def find_file(self, path, saltenv, back=None):
        '''
        
        '''
        path = salt.utils.stringutils.to_unicode(path)
        saltenv = salt.utils.stringutils.to_unicode(saltenv)
        back = self.backends(back)
        kwargs = {}
        fnd = {'path': '',
               'rel': ''}
        if os.path.isabs(path):
            return fnd
        if '../' in path:
            return fnd
        if salt.utils.url.is_escaped(path):
            # don't attempt to find URL query arguments in the path
            path = salt.utils.url.unescape(path)
        else:
            if '?' in path:
                hcomps = path.split('?')
                path = hcomps[0]
                comps = hcomps[1].split('&')
                for comp in comps:
                    if '=' not in comp:
                        # Invalid option, skip it
                        continue
                    args = comp.split('=', 1)
                    kwargs[args[0]] = args[1]

        if 'env' in kwargs:
            # "env" is not supported; Use "saltenv".
            kwargs.pop('env')
        if 'saltenv' in kwargs:
            saltenv = kwargs.pop('saltenv')

        if not isinstance(saltenv, six.string_types):
            saltenv = six.text_type(saltenv)

        for fsb in back:
            fstr = '{0}.find_file'.format(fsb)
            if fstr in self.servers:
                fnd = self.servers[fstr](path, saltenv, **kwargs)
                if fnd.get('path'):
                    fnd['back'] = fsb
                    return fnd
        return fnd