Retrieve output files associated with an entire analysis project.
517394
def _get_files_project(sample, upload_config):
    
    out = [{"path": sample["provenance"]["programs"]}]
    if os.path.exists(tz.get_in(["provenance", "data"], sample) or ""):
        out.append({"path": sample["provenance"]["data"]})
    for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
        if os.path.exists(os.path.join(log.get_log_dir(sample["config"]), fname)):
            out.append({"path": os.path.join(log.get_log_dir(sample["config"]), fname),
                        "type": "external_command_log",
                        "ext": ""})

    if "summary" in sample and sample["summary"].get("project"):
        out.append({"path": sample["summary"]["project"]})
    if "summary" in sample and sample["summary"].get("metadata"):
        out.append({"path": sample["summary"]["metadata"]})
    mixup_check = tz.get_in(["summary", "mixup_check"], sample)
    if mixup_check:
        out.append({"path": sample["summary"]["mixup_check"],
                    "type": "directory", "ext": "mixup_check"})

    report = os.path.join(dd.get_work_dir(sample), "report")
    if utils.file_exists(report):
        out.append({"path": report,
                    "type": "directory", "ext": "report"})

    multiqc = tz.get_in(["summary", "multiqc"], sample)
    if multiqc:
        out.extend(_flatten_file_with_secondary(multiqc, "multiqc"))

    if sample.get("seqcluster", {}):
        out.append({"path": sample["seqcluster"].get("out_dir"),
                    "type": "directory", "ext": "seqcluster"})

    if sample.get("mirge", {}):
        for fn in sample["mirge"]:
            out.append({"path": fn,
                        "dir": "mirge"})

    if sample.get("report", None):
        out.append({"path": os.path.dirname(sample["report"]),
                    "type": "directory", "ext": "seqclusterViz"})

    for x in sample.get("variants", []):
        if "pop_db" in x:
            out.append({"path": x["pop_db"],
                        "type": "sqlite",
                        "variantcaller": x["variantcaller"]})
    for x in sample.get("variants", []):
        if "population" in x:
            pop_db = tz.get_in(["population", "db"], x)
            if pop_db:
                out.append({"path": pop_db,
                            "type": "sqlite",
                            "variantcaller": x["variantcaller"]})
            suffix = "-annotated-decomposed" if tz.get_in(("population", "decomposed"), x) else "-annotated"
            vcfs = _get_project_vcf(x, suffix)
            out.extend([_add_batch(f, sample) for f in vcfs])
    for x in sample.get("variants", []):
        if x.get("validate") and x["validate"].get("grading_summary"):
            out.append({"path": x["validate"]["grading_summary"]})
            break
    sv_project = set([])
    for svcall in sample.get("sv", []):
        if svcall.get("variantcaller") == "seq2c":
            if svcall.get("calls_all") and svcall["calls_all"] not in sv_project:
                out.append({"path": svcall["coverage_all"], "batch": "seq2c", "ext": "coverage", "type": "tsv"})
                out.append({"path": svcall["read_mapping"], "batch": "seq2c", "ext": "read_mapping", "type": "txt"})
                out.append({"path": svcall["calls_all"], "batch": "seq2c", "ext": "calls", "type": "tsv"})
                sv_project.add(svcall["calls_all"])
    if "coverage" in sample:
        cov_db = tz.get_in(["coverage", "summary"], sample)
        if cov_db:
            out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
        all_coverage = tz.get_in(["coverage", "all"], sample)
        if all_coverage:
            out.append({"path": all_coverage, "type": "bed", "ext": "coverage"})

    if dd.get_mirna_counts(sample):
        out.append({"path": dd.get_mirna_counts(sample)})
    if dd.get_isomir_counts(sample):
        out.append({"path": dd.get_isomir_counts(sample)})
    if dd.get_novel_mirna_counts(sample):
        out.append({"path": dd.get_novel_mirna_counts(sample)})
    if dd.get_novel_isomir_counts(sample):
        out.append({"path": dd.get_novel_isomir_counts(sample)})
    if dd.get_combined_counts(sample):
        count_file = dd.get_combined_counts(sample)
        if sample["analysis"].lower() == "scrna-seq":
            out.append({"path": count_file,
                        "type": "mtx"})
            out.append({"path": count_file + ".rownames",
                        "type": "rownames"})
            out.append({"path": count_file + ".colnames",
                        "type": "colnames"})
            out.append({"path": count_file + ".metadata",
                        "type": "metadata"})
            umi_file = os.path.splitext(count_file)[0] + "-dupes.mtx"
            if utils.file_exists(umi_file):
                out.append({"path": umi_file,
                            "type": "mtx"})
                out.append({"path": umi_file + ".rownames",
                            "type": "rownames"})
                out.append({"path": umi_file + ".colnames",
                            "type": "colnames"})
            if dd.get_combined_histogram(sample):
                out.append({"path": dd.get_combined_histogram(sample),
                            "type": "txt"})
            rda = os.path.join(os.path.dirname(count_file), "se.rda")
            if utils.file_exists(rda):
                out.append({"path": rda,
                            "type": "rda"})
        else:
            out.append({"path": dd.get_combined_counts(sample)})
    if dd.get_annotated_combined_counts(sample):
        out.append({"path": dd.get_annotated_combined_counts(sample)})
    if dd.get_combined_fpkm(sample):
        out.append({"path": dd.get_combined_fpkm(sample)})
    if dd.get_combined_fpkm_isoform(sample):
        out.append({"path": dd.get_combined_fpkm_isoform(sample)})
    if dd.get_transcript_assembler(sample):
        out.append({"path": dd.get_merged_gtf(sample)})
    if dd.get_dexseq_counts(sample):
        out.append({"path": dd.get_dexseq_counts(sample)})
        out.append({"path": "%s.ann" % dd.get_dexseq_counts(sample)})
    if dd.get_express_counts(sample):
        out.append({"path": dd.get_express_counts(sample)})
    if dd.get_express_fpkm(sample):
        out.append({"path": dd.get_express_fpkm(sample)})
    if dd.get_express_tpm(sample):
        out.append({"path": dd.get_express_tpm(sample)})
    if dd.get_isoform_to_gene(sample):
        out.append({"path": dd.get_isoform_to_gene(sample)})
    if dd.get_square_vcf(sample):
        out.append({"path": dd.get_square_vcf(sample)})
    if dd.get_sailfish_transcript_tpm(sample):
        out.append({"path": dd.get_sailfish_transcript_tpm(sample)})
    if dd.get_sailfish_gene_tpm(sample):
        out.append({"path": dd.get_sailfish_gene_tpm(sample)})
    if dd.get_tx2gene(sample):
        out.append({"path": dd.get_tx2gene(sample)})
    if dd.get_spikein_counts(sample):
        out.append({"path": dd.get_spikein_counts(sample)})
    transcriptome_dir = os.path.join(dd.get_work_dir(sample), "inputs",
                                     "transcriptome")
    if os.path.exists(transcriptome_dir):
        out.append({"path": transcriptome_dir, "type": "directory",
                    "ext": "transcriptome"})
    return _add_meta(out, config=upload_config)
904980
def analyze_lib(lib_dir, cover_filename, *, ignore_existing=False):
  
  work = {}
  stats = collections.OrderedDict(((k, 0) for k in("files", "albums", "missing covers", "errors")))
  with tqdm.tqdm(desc="Analyzing library",
                 unit="dir",
                 postfix=stats) as progress, \
          tqdm_logging.redirect_logging(progress):
    for rootpath, rel_dirpaths, rel_filepaths in os.walk(lib_dir):
      metadata = analyze_dir(stats,
                             rootpath,
                             rel_filepaths,
                             cover_filename,
                             ignore_existing=ignore_existing)
      progress.set_postfix(stats, refresh=False)
      progress.update(1)
      if all(metadata[:-1]):
        work[rootpath] = metadata[:-1]
  return work
1147244
def collect_outs(self):
        """"""
        # TODO: this should probably live in CacheManager.
        for outfile in self.rule.output_files or []:
            outfile_built = os.path.join(self.buildroot, outfile)
            if not os.path.exists(outfile_built):
                raise error.TargetBuildFailed(
                    self.address, 'Output file is missing: %s' % outfile)

        #git_sha = gitrepo.RepoState().GetRepo(self.address.repo).repo.commit()
        # git_sha is insufficient, and is actually not all that useful.
        # More factors to include in hash:
        # - commit/state of source repo of all dependencies
        #   (or all input files?)
        #   - Actually I like that idea: hash all the input files!
        # - versions of build tools used (?)
        metahash = self._metahash()
        log.debug('[%s]: Metahash: %s', self.address, metahash.hexdigest())
        # TODO: record git repo state and buildoptions in cachemgr
        # TODO: move cachemgr to outer controller(?)
        self.cachemgr.putfile(outfile_built, self.buildroot, metahash)
742311
def output_files(self):
        """"""
        output_list = []

        pm_dict = self.get_dict()
        output_list.append(pm_dict['output_surface'])
        if pm_dict['target_volume'] != 0.0:
            output_list.append(pm_dict['output_surface'] + str(".cell"))

        return output_list
1256346
def get_data_files():
    """"""
    if sys.platform.startswith('linux'):
        if PY3:
            data_files = [('share/applications',
                           ['scripts/condamanager3.desktop']),
                          ('share/pixmaps',
                           ['img_src/condamanager3.png'])]
        else:
            data_files = [('share/applications',
                           ['scripts/condamanager.desktop']),
                          ('share/pixmaps',
                           ['img_src/condamanager.png'])]
    elif os.name == 'nt':
        data_files = [('scripts', ['img_src/conda-manager.ico'])]
    else:
        data_files = []
    return data_files
1046534
function () {
            var outputDir = path.join(copyDir, jalangiRuntimeDir);
            mkdirp.sync(outputDir);
            var copyFile = function (srcFile) {
                if (jalangiRoot) {
                    srcFile = path.join(jalangiRoot, srcFile);
                }
                var outputFile = path.join(outputDir, path.basename(srcFile));
                fs.writeFileSync(outputFile, String(fs.readFileSync(srcFile)));
            };
            instUtil.headerSources.forEach(copyFile);
            if (analyses) {
                analyses.forEach(function (f) {
                    var outputFile = path.join(outputDir, path.basename(f));
                    fs.writeFileSync(outputFile, String(fs.readFileSync(f)));
                });
            }
        }
792239
def collect_results(working_folder):
    
    results = {} 
    working_folder = Path(working_folder)
    output_folders = working_folder/'output_folders'

    for rec_name in os.listdir(output_folders):
        if not os.path.isdir(output_folders / rec_name):
            continue
        # print(rec_name)
        results[rec_name] = {}
        for sorter_name in os.listdir(output_folders / rec_name):
            # print('  ', sorter_name)
            output_folder = output_folders / rec_name / sorter_name
            #~ print(output_folder)
            if not os.path.isdir(output_folder):
                continue
            SorterClass = sorter_dict[sorter_name]
            results[rec_name][sorter_name] = SorterClass.get_result_from_folder(output_folder)

    return results
456269
def _get_input_files(samples, base_dir, tx_out_dir):
    
    in_files = collections.defaultdict(list)
    for data in samples:
        sum_qc = tz.get_in(["summary", "qc"], data, {})
        if sum_qc in [None, "None"]:
            sum_qc = {}
        elif isinstance(sum_qc, six.string_types):
            sum_qc = {dd.get_algorithm_qc(data)[0]: sum_qc}
        elif not isinstance(sum_qc, dict):
            raise ValueError("Unexpected summary qc: %s" % sum_qc)
        for program, pfiles in sum_qc.items():
            if isinstance(pfiles, dict):
                pfiles = [pfiles["base"]] + pfiles.get("secondary", [])
            # CWL: presents output files as single file plus associated secondary files
            elif isinstance(pfiles, six.string_types):
                if os.path.exists(pfiles):
                    pfiles = [os.path.join(basedir, f) for basedir, subdir, filenames in os.walk(os.path.dirname(pfiles)) for f in filenames]
                else:
                    pfiles = []
            in_files[(dd.get_sample_name(data), program)].extend(pfiles)
    staged_files = []
    for (sample, program), files in in_files.items():
        cur_dir = utils.safe_makedir(os.path.join(base_dir, "inputs", sample, program))
        for f in files:
            if _check_multiqc_input(f) and _is_good_file_for_multiqc(f):
                if _in_temp_directory(f) or any([cwlutils.is_cwl_run(d) for d in samples]):
                    staged_f = os.path.join(cur_dir, os.path.basename(f))
                    shutil.copy(f, staged_f)
                    staged_files.append(staged_f)
                else:
                    staged_files.append(f)
    staged_files.extend(get_qsig_multiqc_files(samples))
    # Back compatible -- to migrate to explicit specifications in input YAML
    if not any([cwlutils.is_cwl_run(d) for d in samples]):
        staged_files += ["trimmed", "htseq-count/*summary"]
        # Add in created target_info file
        if os.path.isfile(os.path.join(base_dir, "report", "metrics", "target_info.yaml")):
            staged_files += [os.path.join(base_dir, "report", "metrics", "target_info.yaml")]
    return sorted(list(set(staged_files)))
712307
def system_find_analyses(input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/system/findAnalyses', input_params, always_retry=always_retry, **kwargs)
685285
def collect(self):
        """"""
        for app_name, tools_path in get_apps_tools().items():
            self.stdout.write("Copying files from '{}'.".format(tools_path))

            app_name = app_name.replace('.', '_')

            app_destination_path = os.path.join(self.destination_path, app_name)
            if not os.path.isdir(app_destination_path):
                os.mkdir(app_destination_path)

            for root, dirs, files in os.walk(tools_path):
                for dir_name in dirs:
                    dir_source_path = os.path.join(root, dir_name)
                    dir_destination_path = self.change_path_prefix(
                        dir_source_path, tools_path, self.destination_path, app_name
                    )

                    if not os.path.isdir(dir_destination_path):
                        os.mkdir(dir_destination_path)

                for file_name in files:
                    file_source_path = os.path.join(root, file_name)
                    file_destination_path = self.change_path_prefix(
                        file_source_path, tools_path, self.destination_path, app_name
                    )

                    shutil.copy2(file_source_path, file_destination_path)
140725
def fetch_task_output(self):
        
        # 
        if os.path.isfile(os.path.join(self.tmp_dir, "job.out")):
            with open(os.path.join(self.tmp_dir, "job.out"), "r") as f_out:
                outputs = f_out.readlines()
        else:
            outputs = ''
        return outputs
108736
def collect_results(project, force=False):
    """"""
    if not project.crawlable:
        return project

    now = datetime.datetime.now()

    if (now - project.updated_at).total_seconds() < 4 and (not force):
        return project

    result_paths = []
    if os.path.isdir(project.path_name):
        result_paths.extend(_list_result_paths(project.path_name))

    registered_results = db.session.query(Result.path_name).filter_by(
        project_id=project.id
    ).all()
    registered_paths = {r.path_name for r in registered_results}

    for result_path in result_paths:
        if result_path not in registered_paths:
            _register_result(project.id, result_path)

    project.updated_at = datetime.datetime.now()

    db.session.commit()

    return project
1054402
def _upstream_analysis(self, compile_contexts, classpath_entries):
    """"""
    # Reorganize the compile_contexts by class directory.
    compile_contexts_by_directory = {}
    for compile_context in compile_contexts.values():
      compile_context = self.select_runtime_context(compile_context)
      compile_contexts_by_directory[compile_context.classes_dir.path] = compile_context
    # If we have a compile context for the target, include it.
    for entry in classpath_entries:
      path = entry.path
      if not path.endswith('.jar'):
        compile_context = compile_contexts_by_directory.get(path)
        if not compile_context:
          self.context.log.debug('Missing upstream analysis for {}'.format(path))
        else:
          yield compile_context.classes_dir.path, compile_context.analysis_file
1281599
def analysis_output(self, location: Optional[str] = None) -> None:
        
        try:
            if not location:
                location = self.prompt(
                    "Analysis results: ",
                    history_key="analysis_results",
                    completer=PathCompleter(),
                )
            self.current_analysis_output = AnalysisOutput.from_str(location)
        except AnalysisOutputError as e:
            raise UserError(f"Error loading results: {e}")
353874
def get_project_files():
    
    if is_git_project():
        return get_git_project_files()

    project_files = []
    for top, subdirs, files in os.walk('.'):
        for subdir in subdirs:
            if subdir.startswith('.'):
                subdirs.remove(subdir)

        for f in files:
            if f.startswith('.'):
                continue
            project_files.append(os.path.join(top, f))

    return project_files
242711
def extract_archive(archive_path, destination_path):
    """"""
    tar = tarfile.open(archive_path)
    tar.errorlevel = 1
    tar.extractall(destination_path)
341761
def _extract_archive(archive_fpath, archive_file, archive_namelist, output_dir,
                     force_commonprefix=True, prefix=None,
                     dryrun=False, verbose=not QUIET, overwrite=None):
    
    # force extracted components into a subdirectory if force_commonprefix is
    # on return_path = output_diG
    # FIXMpathE doesn't work right
    if prefix is not None:
        output_dir = join(output_dir, prefix)
        util_path.ensurepath(output_dir)

    archive_basename, ext = split_archive_ext(basename(archive_fpath))
    if force_commonprefix and commonprefix(archive_namelist) == '':
        # use the archivename as the default common prefix
        output_dir = join(output_dir, archive_basename)
        util_path.ensurepath(output_dir)

    for member in archive_namelist:
        (dname, fname) = split(member)
        dpath = join(output_dir, dname)
        util_path.ensurepath(dpath)
        if verbose:
            print('[utool] Unarchive ' + fname + ' in ' + dpath)

        if not dryrun:
            if overwrite is False:
                if exists(join(output_dir, member)):
                    continue
            archive_file.extract(member, path=output_dir)
    return output_dir