turn fetched files into a local repo, make auxiliary files
772401
def make(self):
        
        logger.debug("preparing to add all git files")
        num_added = self.local_repo.add_all_files()
        if num_added:
            self.local_repo.commit("Initial import from Project Gutenberg")

        file_handler = NewFilesHandler(self)
        file_handler.add_new_files()

        num_added = self.local_repo.add_all_files()
        if num_added:
            self.local_repo.commit(
                "Updates Readme, contributing, license files, cover, metadata."
            )
527972
def git_fetch(repo_dir, remote=None, refspec=None, verbose=False, tags=True):
    
    command = ['git', 'fetch']
    if not remote:
        command.append('--all')
    else:
        remote = pipes.quote(remote)
    command.extend(['--update-head-ok'])
    if tags:
        command.append('--tags')
    if verbose:
        command.append('--verbose')
    if remote:
        command.append(remote)
    if refspec:
        command.append(refspec)
    return execute_git_command(command, repo_dir=repo_dir)
14401
function remoteAsync() {
  var generator = this.generator;
  
  var username;
  var repo;
  var branch;
  var refresh;
  var url;

  // With URL and refresh
  if (arguments.length <= 2) {
    url = arguments[0];
    refresh = arguments[1];
  } else {
    username = arguments[0];
    repo = arguments[1];
    branch = arguments[2];
    refresh = arguments[3];
    url = 'https://github.com/' + [username, repo, 'archive', branch].join('/') + '.tar.gz';
  }
  
  return new Promise(function(resolve, reject) {
    parent.remote.call(parent, url, function(err, remote) {
      if (err) {
        reject(err);
      }
      else {
        resolve(remote);
      }
    }, refresh);
  });
}
760885
def deploy_dotfiles(self, dotfiles):
        """"""
        if self.args.server:
            return self.deploy_remote(dotfiles)
        else:
            return self.deploy_local(dotfiles)
979396
def hg_hook(ui, repo, node=None, **kwargs):
    """"""
    seen = set()
    paths = []
    if len(repo):
        for rev in range(repo[node], len(repo)):
            for file_ in repo[rev].files():
                file_ = op.join(repo.root, file_)
                if file_ in seen or not op.exists(file_):
                    continue
                seen.add(file_)
                paths.append(file_)

    options = parse_options()
    setup_logger(options)
    if paths:
        process_paths(options, candidates=paths)
1175971
function processGithubFile (options, callback) {
  var item = options.item
  var key = options.key
  var json = options.json

  function respond (content) {
    if (isFeatureCollection(content)) {
      content.name = key
      content.updated_at = json.updated_at
      return content
    }
    return false
  }

  if (item.truncated) {
    return request({
      url: item.raw_url
    }, function (err, content) {
      if (err) return callback(err)

      callback(null, respond(content))
    })
  }

  try {
    var content = JSON.parse(item.content)
  } catch (e) {
    var msg = 'could not parse file contents of ' + key + ': ' + e.message
    return callback(new Error(msg))
  }

  callback(null, respond(content))
}
158615
def from_github(user_repo_pair, file='plashfile'):
    ""
    from urllib.request import urlopen
    url = 'https://raw.githubusercontent.com/{}/master/{}'.format(
        user_repo_pair, file)
    with utils.catch_and_die([Exception], debug=url):
        resp = urlopen(url)
    plashstr = resp.read()
    return utils.run_write_read(['plash', 'build', '--eval-stdin'],
                                plashstr).decode().rstrip('\n')
492287
def library(repo):
    
    pkg_list, packages = [], ""
    if repo == "sbo":
        if (os.path.isfile(
                _meta_.lib_path + "{0}_repo/SLACKBUILDS.TXT".format(repo))):
            packages = Utils().read_file(_meta_.lib_path + "{0}_repo/"
                                         "SLACKBUILDS.TXT".format(repo))
    else:
        if (os.path.isfile(
                _meta_.lib_path + "{0}_repo/PACKAGES.TXT".format(repo))):
            packages = Utils().read_file(_meta_.lib_path + "{0}_repo/"
                                         "PACKAGES.TXT".format(repo))
    for line in packages.splitlines():
        if repo == "sbo":
            if line.startswith("SLACKBUILD NAME: "):
                pkg_list.append(line[17:].strip())
        elif "local" not in repo:
            if line.startswith("PACKAGE NAME: "):
                pkg_list.append(line[15:].strip())
    if repo == "local":
        pkg_list = find_package("", _meta_.pkg_path)
    return pkg_list
887799
def request_fetch(self, user, repo, request, pull=False, force=False): #pragma: no cover
        '''
        '''
        raise NotImplementedError
741319
def fetch_local_package(self, config):
        
        # Update sys.path & merge in remote configs (if necessary)
        self.update_paths_and_config(config=config,
                                     pkg_dir_name=config['source'],
                                     pkg_cache_dir=os.getcwd())
1051155
def gather_repositories():
    

    for (root, dirs, files) in os.walk('.', topdown=True):
        if '.git' not in dirs:
            continue

        for dir in list(dirs):
            dirs.remove(dir)

        path = os.path.split(root)[1]
        repo = os.path.basename(path)
        yield (repo, root)
168811
def main(unused_argv):
  """"""
  make_dir(FLAGS.raw_dir)
  make_dir(FLAGS.data_dir)

  # Get paths of download/extracted training and evaluation files.
  print("Step 1/4: Downloading data from source")
  train_files = get_raw_files(FLAGS.raw_dir, _TRAIN_DATA_SOURCES)
  eval_files = get_raw_files(FLAGS.raw_dir, _EVAL_DATA_SOURCES)
652377
function(repo, version, hash, meta, outDir) {

    var url, tempRepoDir, packageJSONData, self = this;

    if (meta.vPrefix) {
      version = 'v' + version;
    }

    // Automatically track and cleanup files at exit
    temp.track();

    url = createGitUrl(self.options.baseurl, repo, self.options.reposuffix, self.auth);

    return createTempDir().then(function(tempDir) {
      tempRepoDir = tempDir;
      return exportGitRepo(tempRepoDir, version, url, self.execOpt, self.options.shallowclone);
    }).then(function() {
      return readPackageJSON(tempRepoDir);
    }).then(function(data) {
      packageJSONData = data;
      return moveRepoToOutDir(tempRepoDir, outDir);
    }).then(function() {
      return packageJSONData;
    });
  }
354607
function expandFiles(files, workingDirectory) {
  const promises = [];
  files.forEach((file) => {
    if (
      file.path !== './package.json' && file.path !== './npm-shrinkwrap.json'
    ) {
      promises.push(Promise.resolve(file));
      return;
    }
    findPackageDependencies(workingDirectory, true).forEach((dep) => {
      let resolvedPath;
      try {
        resolvedPath = forwardSlashes(path.relative(
          workingDirectory,
          requireRelative.resolve(dep, workingDirectory),
        ));
        if (!resolvedPath.startsWith('.')) {
          // path.relative won't add "./", but we need it further down the line
          resolvedPath = `./${resolvedPath}`;
        }
      } catch (e) {
        winston.error(`Failed to resolve "${dep}" relative to ${workingDirectory}`);
        return;
      }

      promises.push(lastUpdate.failSafe(resolvedPath, workingDirectory)
        .then(({ mtime }) =>
          Promise.resolve({
            path: resolvedPath,
            mtime,
            packageName: dep,
          })));
    });
  });
  return Promise.all(promises);
}
331345
def fetch_file_handler(unused_build_context, target, fetch, package_dir, tar):
    
    dl_dir = join(package_dir, fetch.name) if fetch.name else package_dir
    fetch_url(fetch.uri,
              join(dl_dir, basename(urlparse(fetch.uri).path)),
              dl_dir)
    tar.add(package_dir, arcname=split_name(target.name))
290612
def to_blob(self, repo):
        """"""
        return Blob(repo, self.binsha, self.mode, self.path)
525757
def get_files_from_github(user_and_repo, filedir, regex, prefer_tags):
    '''  '''
    github_url = f'https://api.github.com/repos/{user_and_repo}'
    args = f'?per_page={N_RESULTS}'
    results = []
    json_data = None

    # Determine the right url
    if not prefer_tags:
        url = f'{github_url}/releases{args}'
        json_data = requests.get(url).json()

    if not json_data:
        url = f'{github_url}/tags{args}'
        json_data = requests.get(url).json()

    logger.debug(f'[+] Using {url} as base url.')

    # Get all the releases/tags available
    results = json_data
    for n_page in range(2, 1000):
        logger.debug(f'[+] Requesting page {n_page} of releases/tags ..')
        json_data = requests.get(f'{url}&page={n_page}').json()
        results += json_data

        if len(json_data) < N_RESULTS:
            break

    if not results:
        raise ValueError(f'No releases/tags for {user_and_repo}')

    directories = {}
    for result in results:
        name = result['name']
        m = re.match(regex, name)
        if not m:
            continue

        name = m.groups()[0]

        logger.debug(f'[+] Downloading zip file for {name} version ..')

        # Download zip file and extract in temporary directory
        zip_url = result['zipball_url']
        zf = requests.get(zip_url, allow_redirects=True)
        z = zipfile.ZipFile(io.BytesIO(zf.content))
        output_dir = f'{filedir}/{z.namelist()[0]}'

        z.extractall(path=filedir)
        directories[name] = output_dir

    return directories