Upload results of processing from an analysis pipeline sample.
346805
def from_sample(sample):
    
    upload_config = sample.get("upload")
    if upload_config:
        approach = _approaches[upload_config.get("method", "filesystem")]
        for finfo in _get_files(sample):
            approach.update_file(finfo, sample, upload_config)
    return [[sample]]
656807
def ProcessResponse(self, client_id, response):
    """"""
    precondition.AssertType(client_id, Text)
    downsampled = rdf_client_stats.ClientStats.Downsampled(response)

    if data_store.AFF4Enabled():
      urn = rdf_client.ClientURN(client_id).Add("stats")

      with aff4.FACTORY.Create(
          urn, aff4_stats.ClientStats, token=self.token, mode="w") as stats_fd:
        # Only keep the average of all values that fall within one minute.
        stats_fd.AddAttribute(stats_fd.Schema.STATS, downsampled)

    if data_store.RelationalDBEnabled():
      data_store.REL_DB.WriteClientStats(client_id, downsampled)

    return downsampled
724591
def run_analysis(self, argv):
        """"""
        args = self._parser.parse_args(argv)


        sedfile = args.sed_file

        if is_not_null(args.config):
            configfile = os.path.join(os.path.dirname(sedfile), args.config)
        else:
            configfile = os.path.join(os.path.dirname(sedfile), 'config.yaml')

        nbins = _get_enum_bins(configfile)

        first = args.seed
        last = first + args.nsims
        flist = [sedfile.replace("_SEED.fits", "_%06i.fits" % seed)
                 for seed in range(first, last)]
        outfile = args.outfile
        summaryfile = args.summaryfile


        outtable = fill_output_table(
            flist, "SED", CollectSED.collist, nbins=nbins)

        if is_not_null(outfile):
            outtable.write(outfile)

        if is_not_null(summaryfile):
            summary = summarize_sed_results(outtable)
            summary.write(summaryfile)
1342432
def collect_api_results(input_data, url, headers, api, batch_size, kwargs):
    
    if batch_size:
        results = []
        for batch in batched(input_data, size=batch_size):
            try:
                result = send_request(batch, api, url, headers, kwargs)
                if isinstance(result, list):
                    results.extend(result)
                else:
                    results.append(result)
            except IndicoError as e:
                # Log results so far to file
                timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
                filename = "indico-{api}-{timestamp}.json".format(
                    api=api,
                    timestamp=timestamp
                )
                if sys.version_info > (3, 0):
                    json.dump(results, open(filename, mode='w', encoding='utf-8'), cls=NumpyEncoder)
                else:
                    json.dump(results, open(filename, mode='w'), cls=NumpyEncoder)
                raise BatchProcessingError(
                    "The following error occurred while processing your data: `{err}` "
                    "Partial results have been saved to {filename}".format(
                        err=e,
                        filename=os.path.abspath(filename)
                    )
                )
        return results
    else:
        return send_request(input_data, api, url, headers, kwargs)
1358405
public void submit(final Collection<Sample> samples) {
        for (final SampleProcessor processor : m_processors) {
            m_executor.execute(new Runnable() {

                @Override
                public void run() {
                    processor.submit(samples);
                }
            });
        }
    }
645036
def add_analysis(self, analysis):
        
        analysis_object = api.get_object(analysis)
        result = analysis_object.getResult()
        results_range = analysis_object.getResultsRange()
        range_result = results_range.get('result', None)
        range_min = results_range.get('min', None)
        range_max = results_range.get('max', None)
        # All them must be floatable
        for value in [result, range_result, range_min, range_max]:
            if not api.is_floatable(value):
                return
        cap_date = analysis_object.getResultCaptureDate()
        cap_date = api.is_date(cap_date) and \
                   cap_date.strftime('%Y-%m-%d %I:%M %p') or ''
        if not cap_date:
            return

        # Create json
        ref_sample_id = analysis_object.getSample().getId()
        as_keyword = analysis_object.getKeyword()
        as_name = analysis_object.Title()
        as_ref = '{} ({})'.format(as_name, as_keyword)
        as_rows = self.analyses_dict.get(as_ref, {})
        an_rows = as_rows.get(ref_sample_id, [])
        an_rows.append({
            'date': cap_date,
            'target': api.to_float(range_result),
            'upper': api.to_float(range_max),
            'lower': api.to_float(range_min),
            'result': api.to_float(result),
            'unit': analysis_object.getUnit(),
            'id': api.get_uid(analysis_object)
        })
        as_rows[ref_sample_id] = an_rows
        self.analyses_dict[as_ref] = as_rows
366086
protected final function GatherOutput()
    {
        \header('Content-Type: application/json');
        try
        {
            $this->output = json_encode($this->result);
        }
        catch (\Exception $ex)
        {
            $this->AttachException($ex);
        }
    }
584414
def analyze(self, handle, filename):
        
        # multipart post files.
        files = {"file" : (filename, handle)}

        # ensure the handle is at offset 0.
        handle.seek(0)

        response = self._request("/submit/file", method='POST', files=files)

        try:
            if response.status_code == 201:
                # good response
                return response.json()['job_id']
            else:
                raise sandboxapi.SandboxError("api error in analyze: {r}".format(r=response.content.decode('utf-8')))
        except (ValueError, KeyError) as e:
            raise sandboxapi.SandboxError("error in analyze: {e}".format(e=e))
1244009
def run_analysis(self, argv):
        """"""
        args = self._parser.parse_args(argv)

        exttype = splitext(args.infile)[-1]
        if exttype in ['.fits', '.npy']:
            castro_data = CastroData.create_from_sedfile(args.infile)
        elif exttype in ['.yaml']:
            castro_data = CastroData.create_from_yamlfile(args.infile)
        else:
            raise ValueError("Can not read file type %s for SED" % extype)

        ylims = [1e-8, 1e-5]

        plot = plotCastro(castro_data, ylims)
        if args.outfile:
            plot[0].savefig(args.outfile)
548003
def getoutputfiles(self, loadmetadata=True, client=None,requiremetadata=False):
        """"""
        for outputfilename, outputtemplate in self.outputpairs():
            yield CLAMOutputFile(self.projectpath, outputfilename, loadmetadata,client,requiremetadata), outputtemplate
1203868
def process(self, pipeline):
        """"""
        output = []
        for entry in pipeline:
            key = list(entry.keys())[0]
            # an environment block can be repeated
            if key == "env":
                self.data.env_list[0].update(entry[key])
                self.logger.debug("Updating environment at level 0 with %s",
                                  self.data.env_list[0])
                continue

            # after validation it can't be anything else but a stage
            # and the title is inside the round brackets:
            stage = Stage(self, re.match(r"stage\((?P<title>.*)\)", key).group("title"))
            result = stage.process(entry[key])
            output += result['output']
            if not result['success']:
                return {'success': False, 'output': output}

        # logging the output of the cleanup shell when registered
        for line in self.cleanup():
            output.append(line)
            self.logger.info(" | %s", line)

        self.event.succeeded()
        return {'success': True, 'output': output}
978316
def analyze(self, output_file=None):
        

        fr = None
        for path, code, line, char, desc in self.parser.parse():

            # Create a new FileResult and register it if we have changed to a new file
            if path not in self.files:
                # Update statistics
                if fr:
                    self.update_stats(fr)
                fr = FileResult(path)
                self.files[path] = fr

            # Add line to the FileResult
            fr.add_error(code, line, char, desc)

        # Add final FileResult to statistics, if any were parsed
        if fr:
            self.update_stats(fr)

        # Generate HTML file
        self.generate(output_file=output_file)
50296
def reporter(self):
        
        # Initialise variables
        combinedrow = ''
        reportdirset = set()
        # Populate a set of all the report directories to use. A standard analysis will only have a single report
        # directory, while pipeline analyses will have as many report directories as there are assembled samples
        for sample in self.metadata:
            # Ignore samples that lack a populated reportdir attribute
            if sample[self.analysistype].reportdir != 'NA':
                make_path(sample[self.analysistype].reportdir)
                # Add to the set - I probably could have used a counter here, but I decided against it
                reportdirset.add(sample[self.analysistype].reportdir)
        # Create a report for each sample from :self.resultprofile
        for sample in self.metadata:
            if sample[self.analysistype].reportdir != 'NA':
                if type(sample[self.analysistype].allelenames) == list:
                    # Populate the header with the appropriate data, including all the genes in the list of targets
                    row = 'Strain,Genus,SequenceType,Matches,{},\n' \
                        .format(','.join(sorted(sample[self.analysistype].allelenames)))
                    # Set the sequence counter to 0. This will be used when a sample has multiple best sequence types.
                    # The name of the sample will not be written on subsequent rows in order to make the report clearer
                    seqcount = 0
                    # Iterate through the best sequence types for the sample (only occurs if update profile is disabled)
                    for seqtype in self.resultprofile[sample.name]:
                        """
                        {
                            "OLF15230-1_2015-SEQ-0783": {
                                "1000004_CFIA": {
                                    "7": {
                                        "dnaE": {
                                            "47": "100.00"
                                        },
                                        "dtdS": {
                                            "19": "100.00"
                                        },
                                        "gyrB": {
                                            "359": "100.00"
                                        },
                                        "pntA": {
                                            "50": "100.00"
                                        },
                                        "pyrC": {
                                            "143": "100.00"
                                        },
                                        "recA": {
                                            "31": "100.00"
                                        },
                                        "tnaA": {
                                            "26": "100.00"
                                        }
                                    }
                                }
                            }
                        }
                        """
                        # Becomes
                        """
                        Strain,SequenceType,Matches,dnaE,gyrB,recA,dtdS,pntA,pyrC,tnaA
                        OLF15230-1_2015-SEQ-0783,1000004_CFIA,7,26 (100.00%),359 (100.00%),31 (100.00%),50 (100.00%),
                            19 (100.00%),47 (100.00%),143 (100.00%)
                        """
                        sample[self.analysistype].sequencetype = seqtype
                        # The number of matches to the profile
                        matches = list(self.resultprofile[sample.name][seqtype].keys())[0]
                        # If this is the first of one or more sequence types, include the sample name
                        if seqcount == 0:
                            row += '{},{},{},{},'.format(sample.name, sample.general.referencegenus, seqtype, matches)
                        # Otherwise, skip the sample name
                        else:
                            row += ',,{},{},'.format(seqtype, matches)
                        # Iterate through all the genes present in the analyses for the sample
                        for gene in sorted(sample[self.analysistype].allelenames):
                            # refallele = self.profiledata[self.analysistype][seqtype][gene]
                            refallele = sample[self.analysistype].profiledata[seqtype][gene]
                            # Set the allele and percent id from the dictionary's keys and values, respectively
                            allele = list(self.resultprofile[sample.name][seqtype][matches][gene].keys())[0]
                            percentid = list(self.resultprofile[sample.name][seqtype][matches][gene].values())[0]
                            if refallele and refallele != allele:
                                if 0 < float(percentid) < 100:
                                    row += '{} ({:.2f}%),'.format(allele, float(percentid))
                                else:
                                    row += '{} ({}),'.format(allele, refallele)
                            else:
                                # Add the allele and % id to the row (only add the percent identity if it is not 100%)
                                if 0 < float(percentid) < 100:
                                    row += '{} ({:.2f}%),'.format(allele, float(percentid))
                                else:
                                    row += '{},'.format(allele)
                            self.referenceprofile[sample.name][gene] = allele
                        # Add a newline
                        row += '\n'
                        # Increment the number of sequence types observed for the sample
                        seqcount += 1
                    combinedrow += row
                    # If the length of the # of report directories is greater than 1 (script is being run as part of
                    # the assembly pipeline) make a report for each sample
                    if self.pipeline:
                        # Open the report
                        with open('{}{}_{}.csv'.format(sample[self.analysistype].reportdir, sample.name,
                                                       self.analysistype), 'w') as report:
                            # Write the row to the report
                            report.write(row)
                dotter()
            # Create the report folder
            make_path(self.reportpath)
            # Create the report containing all the data from all samples
            if self.pipeline:
                with open('{}{}.csv'.format(self.reportpath, self.analysistype), 'w') \
                        as combinedreport:
                    # Write the results to this report
                    combinedreport.write(combinedrow)
            else:
                with open('{}{}_{:}.csv'.format(self.reportpath, self.analysistype, time.strftime("%Y.%m.%d.%H.%M.%S")),
                          'w') as combinedreport:
                    # Write the results to this report
                    combinedreport.write(combinedrow)
            # Remove the raw results csv
            [os.remove(rawresults) for rawresults in glob('{}*rawresults*'.format(self.reportpath))]
517394
def _get_files_project(sample, upload_config):
    
    out = [{"path": sample["provenance"]["programs"]}]
    if os.path.exists(tz.get_in(["provenance", "data"], sample) or ""):
        out.append({"path": sample["provenance"]["data"]})
    for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
        if os.path.exists(os.path.join(log.get_log_dir(sample["config"]), fname)):
            out.append({"path": os.path.join(log.get_log_dir(sample["config"]), fname),
                        "type": "external_command_log",
                        "ext": ""})

    if "summary" in sample and sample["summary"].get("project"):
        out.append({"path": sample["summary"]["project"]})
    if "summary" in sample and sample["summary"].get("metadata"):
        out.append({"path": sample["summary"]["metadata"]})
    mixup_check = tz.get_in(["summary", "mixup_check"], sample)
    if mixup_check:
        out.append({"path": sample["summary"]["mixup_check"],
                    "type": "directory", "ext": "mixup_check"})

    report = os.path.join(dd.get_work_dir(sample), "report")
    if utils.file_exists(report):
        out.append({"path": report,
                    "type": "directory", "ext": "report"})

    multiqc = tz.get_in(["summary", "multiqc"], sample)
    if multiqc:
        out.extend(_flatten_file_with_secondary(multiqc, "multiqc"))

    if sample.get("seqcluster", {}):
        out.append({"path": sample["seqcluster"].get("out_dir"),
                    "type": "directory", "ext": "seqcluster"})

    if sample.get("mirge", {}):
        for fn in sample["mirge"]:
            out.append({"path": fn,
                        "dir": "mirge"})

    if sample.get("report", None):
        out.append({"path": os.path.dirname(sample["report"]),
                    "type": "directory", "ext": "seqclusterViz"})

    for x in sample.get("variants", []):
        if "pop_db" in x:
            out.append({"path": x["pop_db"],
                        "type": "sqlite",
                        "variantcaller": x["variantcaller"]})
    for x in sample.get("variants", []):
        if "population" in x:
            pop_db = tz.get_in(["population", "db"], x)
            if pop_db:
                out.append({"path": pop_db,
                            "type": "sqlite",
                            "variantcaller": x["variantcaller"]})
            suffix = "-annotated-decomposed" if tz.get_in(("population", "decomposed"), x) else "-annotated"
            vcfs = _get_project_vcf(x, suffix)
            out.extend([_add_batch(f, sample) for f in vcfs])
    for x in sample.get("variants", []):
        if x.get("validate") and x["validate"].get("grading_summary"):
            out.append({"path": x["validate"]["grading_summary"]})
            break
    sv_project = set([])
    for svcall in sample.get("sv", []):
        if svcall.get("variantcaller") == "seq2c":
            if svcall.get("calls_all") and svcall["calls_all"] not in sv_project:
                out.append({"path": svcall["coverage_all"], "batch": "seq2c", "ext": "coverage", "type": "tsv"})
                out.append({"path": svcall["read_mapping"], "batch": "seq2c", "ext": "read_mapping", "type": "txt"})
                out.append({"path": svcall["calls_all"], "batch": "seq2c", "ext": "calls", "type": "tsv"})
                sv_project.add(svcall["calls_all"])
    if "coverage" in sample:
        cov_db = tz.get_in(["coverage", "summary"], sample)
        if cov_db:
            out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
        all_coverage = tz.get_in(["coverage", "all"], sample)
        if all_coverage:
            out.append({"path": all_coverage, "type": "bed", "ext": "coverage"})

    if dd.get_mirna_counts(sample):
        out.append({"path": dd.get_mirna_counts(sample)})
    if dd.get_isomir_counts(sample):
        out.append({"path": dd.get_isomir_counts(sample)})
    if dd.get_novel_mirna_counts(sample):
        out.append({"path": dd.get_novel_mirna_counts(sample)})
    if dd.get_novel_isomir_counts(sample):
        out.append({"path": dd.get_novel_isomir_counts(sample)})
    if dd.get_combined_counts(sample):
        count_file = dd.get_combined_counts(sample)
        if sample["analysis"].lower() == "scrna-seq":
            out.append({"path": count_file,
                        "type": "mtx"})
            out.append({"path": count_file + ".rownames",
                        "type": "rownames"})
            out.append({"path": count_file + ".colnames",
                        "type": "colnames"})
            out.append({"path": count_file + ".metadata",
                        "type": "metadata"})
            umi_file = os.path.splitext(count_file)[0] + "-dupes.mtx"
            if utils.file_exists(umi_file):
                out.append({"path": umi_file,
                            "type": "mtx"})
                out.append({"path": umi_file + ".rownames",
                            "type": "rownames"})
                out.append({"path": umi_file + ".colnames",
                            "type": "colnames"})
            if dd.get_combined_histogram(sample):
                out.append({"path": dd.get_combined_histogram(sample),
                            "type": "txt"})
            rda = os.path.join(os.path.dirname(count_file), "se.rda")
            if utils.file_exists(rda):
                out.append({"path": rda,
                            "type": "rda"})
        else:
            out.append({"path": dd.get_combined_counts(sample)})
    if dd.get_annotated_combined_counts(sample):
        out.append({"path": dd.get_annotated_combined_counts(sample)})
    if dd.get_combined_fpkm(sample):
        out.append({"path": dd.get_combined_fpkm(sample)})
    if dd.get_combined_fpkm_isoform(sample):
        out.append({"path": dd.get_combined_fpkm_isoform(sample)})
    if dd.get_transcript_assembler(sample):
        out.append({"path": dd.get_merged_gtf(sample)})
    if dd.get_dexseq_counts(sample):
        out.append({"path": dd.get_dexseq_counts(sample)})
        out.append({"path": "%s.ann" % dd.get_dexseq_counts(sample)})
    if dd.get_express_counts(sample):
        out.append({"path": dd.get_express_counts(sample)})
    if dd.get_express_fpkm(sample):
        out.append({"path": dd.get_express_fpkm(sample)})
    if dd.get_express_tpm(sample):
        out.append({"path": dd.get_express_tpm(sample)})
    if dd.get_isoform_to_gene(sample):
        out.append({"path": dd.get_isoform_to_gene(sample)})
    if dd.get_square_vcf(sample):
        out.append({"path": dd.get_square_vcf(sample)})
    if dd.get_sailfish_transcript_tpm(sample):
        out.append({"path": dd.get_sailfish_transcript_tpm(sample)})
    if dd.get_sailfish_gene_tpm(sample):
        out.append({"path": dd.get_sailfish_gene_tpm(sample)})
    if dd.get_tx2gene(sample):
        out.append({"path": dd.get_tx2gene(sample)})
    if dd.get_spikein_counts(sample):
        out.append({"path": dd.get_spikein_counts(sample)})
    transcriptome_dir = os.path.join(dd.get_work_dir(sample), "inputs",
                                     "transcriptome")
    if os.path.exists(transcriptome_dir):
        out.append({"path": transcriptome_dir, "type": "directory",
                    "ext": "transcriptome"})
    return _add_meta(out, config=upload_config)
943987
def generate_parallel(samples, run_parallel):
    
    to_analyze, extras = _split_samples_by_qc(samples)
    qced = run_parallel("pipeline_summary", to_analyze)
    samples = _combine_qc_samples(qced) + extras
    qsign_info = run_parallel("qsignature_summary", [samples])
    metadata_file = _merge_metadata([samples])
    summary_file = write_project_summary(samples, qsign_info)
    out = []
    for data in samples:
        if "summary" not in data[0]:
            data[0]["summary"] = {}
        data[0]["summary"]["project"] = summary_file
        data[0]["summary"]["metadata"] = metadata_file
        if qsign_info:
            data[0]["summary"]["mixup_check"] = qsign_info[0]["out_dir"]
        out.append(data)
    out = _add_researcher_summary(out, summary_file)
    # MultiQC must be run after all file outputs are set:
    return [[utils.to_single_data(d)] for d in run_parallel("multiqc_summary", [out])]
1276465
def pipeline_summary(data):
    
    data = utils.to_single_data(data)
    work_bam = dd.get_align_bam(data) or dd.get_work_bam(data)
    if not work_bam or not work_bam.endswith(".bam"):
        work_bam = None
    if dd.get_ref_file(data):
        if work_bam or (tz.get_in(["config", "algorithm", "kraken"], data)):  # kraken doesn't need bam
            logger.info("QC: %s %s" % (dd.get_sample_name(data), ", ".join(dd.get_algorithm_qc(data))))
            work_data = cwlutils.unpack_tarballs(utils.deepish_copy(data), data)
            data["summary"] = _run_qc_tools(work_bam, work_data)
            if (len(dd.get_algorithm_qc(data)) == 1 and "output_cwl_keys" in data):
                data["summary"]["qc"] = data["summary"]["qc"].get(dd.get_algorithm_qc(data)[0])
    return [[data]]
174415
def export_pipeline(conf, args):
    """"""
    # Export the source pipeline and save it to file
    src = conf.config['instances'][args.src_instance]
    src_url = api.build_pipeline_url(build_instance_url(src))
    src_auth = tuple([conf.creds['instances'][args.src_instance]['user'],
                      conf.creds['instances'][args.src_instance]['pass']])
    verify_ssl = src.get('verify_ssl', True)
    export_json = api.export_pipeline(src_url, args.src_pipeline_id, src_auth, verify_ssl)
    with open(args.out, 'w') as outFile:
        outFile.write(json.dumps(export_json, indent=4, sort_keys=False))
        return (0, '')