Compares the fetch entries with the files actually
           in the payload, and returns a list of all the files
           that still need to be fetched.
510227
def compare_fetch_with_fs(self):
        

        files_on_fs = set(self.payload_files())
        files_in_fetch = set(self.files_to_be_fetched())

        return list(files_in_fetch - files_on_fs)
79793
@Override
    protected boolean fetchMore () throws CIFSException {
        this.nextRequest.reset(this.response.getResumeKey(), this.response.getLastName());
        this.response.reset();
        try {
            getTreeHandle().send(this.nextRequest, this.response);
            return this.response.getStatus() != NtStatus.NT_STATUS_NO_MORE_FILES;
        }
        catch ( SmbException e ) {
            if ( e.getNtStatus() == NtStatus.NT_STATUS_NO_MORE_FILES ) {
                log.debug("No more entries", e);
                return false;
            }
            throw e;
        }
    }
914615
def gather_files(self):
        
        self.all_files = utils.files_in_path(self.paths["role"])

        return len(self.all_files)
1144356
def fetch_archive(savedir, url, fname, force=False):
    """"""
    dest = pjoin(savedir, fname)
    if os.path.exists(dest) and not force:
        info("already have %s" % fname)
        return dest
    info("fetching %s into %s" % (url, savedir))
    if not os.path.exists(savedir):
        os.makedirs(savedir)
    req = urlopen(url)
    with open(dest, 'wb') as f:
        f.write(req.read())
    return dest
713053
def fetch(*args, **kwargs):
    
    data = kwargs.get('data', None)
    files = kwargs.get('files', {})

    if data and isinstance(data, (basestring, dict)) or files:
        return post(*args, **kwargs)
    return get(*args, **kwargs)
939632
def get_files(self):
        

        method, url = get_URL('get')
        payload = {
            'apikey': self.session.cookies.get('apikey'),
            'logintoken': self.session.cookies.get('logintoken'),
            'transferid': self.transfer_id,
            }

        res = getattr(self.session, method)(url, params=payload)

        if res.status_code == 200:
            transfer_data = res.json()['transfer']
            files = transfer_data['files']

            for file_data in files:
                self._files.append(file_data)

            return self.files

        hellraiser(res)
534101
def fetch(self, is_dl_forced=False):
        

        (files_to_download, ftp) = self._get_file_list(
            self.files['anat_entity']['path'],
            self.files['anat_entity']['pattern'])

        LOG.info(
            'Will Check \n%s\nfrom %s',
            '\n'.join(list(files_to_download)), ftp.getwelcome())

        for dlname in files_to_download:
            localfile = '/'.join((self.rawdir, dlname))
            info = ftp.sendcmd("MLST {}".format(dlname))   # fetch remote file stats
            info = info.split('\n')[1].strip()              # drop pre & post script
            info = info.split(';')                          # partition fields
            info = [item.strip() for item in info[:-1]]     # cleanup an drop final name
            info = [item.split('=') for item in info]       # make pairs
            info = {item[0]: item[1] for item in info}      # transform list to dict

            LOG.info(
                '%s\n'
                'Remote File Size: %i\n'
                'Remote timestamp: %s',
                dlname, int(info['size']),
                self._convert_ftp_time_to_iso(info['modify']))

            if not os.path.exists(localfile) or is_dl_forced or \
                    self.checkIfRemoteIsNewer(
                        localfile, int(info['size']), info['modify']):

                LOG.info("Fetching %s", dlname)
                LOG.info("Writing to %s", localfile)

                ftp.retrbinary('RETR {}'.format(dlname), open(localfile, 'wb').write)
                remote_dt = Bgee._convert_ftp_time_to_iso(info['modify'])
                os.utime(
                    localfile,
                    (time.mktime(remote_dt.timetuple()),
                     time.mktime(remote_dt.timetuple())))
        ftp.quit()

        return
1252609
async function getFileList(dir, fileList) {
  return new Promise(async function (resolve, reject) {  
  fileList = fileList || [];
  var filesIndex = await io.getFiles(dir, null);   
  var files = io.absoluteFiles(filesIndex);
  files.forEach(function (file) {
    fileList.push(file);
  });
  resolve(fileList);
});
}
342686
def matchingfiles(self, projectpath):
        """"""
        results = []

        if projectpath[-1] == '/':
            inputpath = projectpath + 'input/'
        else:
            inputpath = projectpath + '/input/'

        for linkf,realf in clam.common.util.globsymlinks(inputpath + '/.*.INPUTTEMPLATE.' + self.id + '.*'):
            seqnr = int(linkf.split('.')[-1])
            results.append( (seqnr, realf[len(inputpath):], self) )
        results = sorted(results)
        if self.unique and len(results) != 1:
            return []
        else:
            return results
483497
async def fetchall(self):
        
        rows = []
        while True:
            row = await self.fetchone()
            if row is None:
                break
            rows.append(row)
        return rows
397871
def _fetch_events_files_on_disk(self):
    
    all_files = tf.io.gfile.listdir(self._events_directory)
    relevant_files = [
        file_name for file_name in all_files
        if _DEBUGGER_EVENTS_FILE_NAME_REGEX.match(file_name)
    ]
    return sorted(relevant_files, key=self._obtain_file_index)
175431
def list_files(self, project):
        
        path = "/projects/{}/files".format(project.id)
        res = yield from self.http_query("GET", path, timeout=120)
        return res.json
502604
def _get_payload(self, key):
        
        path = self._path(key)

        # If the file doesn't exists, we obviously can't return the cache so we will
        # just return null. Otherwise, we'll get the contents of the file and get
        # the expiration UNIX timestamps from the start of the file's contents.
        if not os.path.exists(path):
            return {'data': None, 'time': None}

        with open(path, 'rb') as fh:
            contents = fh.read()

        expire = int(contents[:10])

        # If the current time is greater than expiration timestamps we will delete
        # the file and return null. This helps clean up the old files and keeps
        # this directory much cleaner for us as old files aren't hanging out.
        if round(time.time()) >= expire:
            self.forget(key)

            return {'data': None, 'time': None}

        data = self.unserialize(contents[10:])

        # Next, we'll extract the number of minutes that are remaining for a cache
        # so that we can properly retain the time for things like the increment
        # operation that may be performed on the cache. We'll round this out.
        time_ = math.ceil((expire - round(time.time())) / 60.)

        return {'data': data, 'time': time_}
1017263
def retrieve_files():
    

    all_files = []
    for location in cfg.CONF.locations or []:
        # if local path then make sure it is absolute
        if not location.startswith('\\'):
            location = os.path.abspath(os.path.expanduser(location))

        LOG.debug('searching [%s]', location)
        for root, _, files in os.walk(location):
            LOG.debug('found file(s) %s', files)
            for name in files:
                filepath = os.path.join(root, name)
                if (os.access(filepath, os.R_OK) and
                        not _is_blacklisted_filename(filepath) and
                        _is_valid_extension(os.path.splitext(name)[1])):
                    all_files.append(filepath)

    return all_files
837740
function(localHashes, files, done) {
        fetchRemoteHashes(function(err, remoteHashes) {
          done(err, localHashes, remoteHashes, files);
        });
      }
923564
function collect(p, files, matches) {
  if (!files) files = []

  matches = matches || function (p) {
    return p.match(/\.js$/)
  }

  if (fs.statSync(p).isDirectory()) {
    fs.readdirSync(p).forEach(function (item) {
      collect(path.join(p, item), files, matches)
    })
  } else if (matches(p)) {
    files.push(p)
  }

  return files
}
48787
def fetch(self, output_path, last_etag=None, lastmodified=None, progress_bar=False):
        
        req_headers = {}
        usr_agent = get_user_agent()
        if usr_agent is not None:
            req_headers['User-Agent'] = usr_agent

        if last_etag is not None:
            req_headers['If-None-Match'] = '"' + last_etag + '"'
        if lastmodified is not None:
            req_headers['If-Modified-Since'] = lastmodified

        response = requests.get(self.url, headers=req_headers, stream=progress_bar)

        if response.status_code == 304:
            # Conditions didn't match, no content returned
            return None
        if response.status_code != 200:
            raise requests.HTTPError

        with open(output_path, 'wb') as f:
            if progress_bar:
                total_size = long(response.headers.get('content-length'))
                for chunk in progress.bar(response.iter_content(chunk_size=self.chunk_size),
                                          expected_size=(total_size / self.chunk_size) + 1):
                    if chunk:
                        f.write(chunk)
                        f.flush()
            else:
                f.write(response.content)

        headers = normalize_headers(response.headers)

        if 'etag' not in headers:
            headers['etag'] = fs_util.calc_file_md5(output_path)

        return headers