%prog report ksfile

    generate a report given a Ks result file (as produced by synonymous_calc.py).
    describe the median Ks, Ka values, as well as the distribution in stem-leaf plot
164075
def report(args):
    '''
    
    '''
    from jcvi.utils.cbook import SummaryStats
    from jcvi.graphics.histogram import stem_leaf_plot

    p = OptionParser(report.__doc__)
    p.add_option("--pdf", default=False, action="store_true",
            help="Generate graphic output for the histogram [default: %default]")
    p.add_option("--components", default=1, type="int",
            help="Number of components to decompose peaks [default: %default]")
    add_plot_options(p)
    opts, args, iopts = p.set_image_options(args, figsize="5x5")

    if len(args) !=  1:
        sys.exit(not p.print_help())

    ks_file, = args
    data = KsFile(ks_file)
    ks_min = opts.vmin
    ks_max = opts.vmax
    bins = opts.bins

    for f in fields.split(",")[1:]:
        columndata = [getattr(x, f) for x in data]
        ks = ("ks" in f)
        if not ks:
            continue

        columndata = [x for x in columndata if ks_min <= x <= ks_max]

        st = SummaryStats(columndata)
        title = "{0} ({1}): ".format(descriptions[f], ks_file)
        title += "Median:{0:.3f} (1Q:{1:.3f}|3Q:{2:.3f}||".\
                format(st.median, st.firstq, st.thirdq)
        title += "Mean:{0:.3f}|Std:{1:.3f}||N:{2})".\
                format(st.mean, st.sd, st.size)

        tbins = (0, ks_max, bins) if ks else (0, .6, 10)
        digit = 2 if (ks_max * 1. / bins) < .1 else 1
        stem_leaf_plot(columndata, *tbins, digit=digit, title=title)

    if not opts.pdf:
        return

    components = opts.components
    data = [x.ng_ks for x in data]
    data = [x for x in data if ks_min <= x <= ks_max]

    fig = plt.figure(1, (iopts.w, iopts.h))
    ax = fig.add_axes([.12, .1, .8, .8])
    kp = KsPlot(ax, ks_max, opts.bins, legendp=opts.legendp)
    kp.add_data(data, components, fill=opts.fill, fitted=opts.fit)
    kp.draw(title=opts.title)
822718
def Main():
  
  argument_parser = argparse.ArgumentParser(description=(
      'Plots memory usage from profiling data.'))

  argument_parser.add_argument(
      '--output', dest='output_file', type=str, help=(
          'path of the output file to write the graph to instead of using '
          'interactive mode. The output format deduced from the extension '
          'of the filename.'))

  argument_parser.add_argument(
      'profile_path', type=str, help=(
          'path to the directory containing the profiling data.'))

  options = argument_parser.parse_args()

  if not os.path.isdir(options.profile_path):
    print('No such directory: {0:s}'.format(options.profile_path))
    return False

  names = ['time', 'queued', 'processing', 'to_merge', 'abandoned', 'total']

  glob_expression = os.path.join(options.profile_path, 'task_queue-*.csv.gz')
  for csv_file_name in glob.glob(glob_expression):
    data = numpy.genfromtxt(
        csv_file_name, delimiter='\t', dtype=None, encoding='utf-8',
        names=names, skip_header=1)

    pyplot.plot(data['time'], data['queued'], label='queued')
    pyplot.plot(data['time'], data['processing'], label='processing')
    pyplot.plot(data['time'], data['to_merge'], label='to merge')
    pyplot.plot(data['time'], data['abandoned'], label='abandoned')

  pyplot.title('Number of tasks over time')

  pyplot.xlabel('Time')
  pyplot.xscale('linear')

  pyplot.ylabel('Number of tasks')
  pyplot.yscale('linear')

  pyplot.legend()

  if options.output_file:
    pyplot.savefig(options.output_file)
  else:
    pyplot.show()

  return True
847266
def print_results_from_args(args: argparse.Namespace):
    
    path = args.path
    metrics_name = args.metrics_filename
    keys = args.keys

    results_dict = {}
    for root, _, files in os.walk(path):

        if metrics_name in files:
            full_name = os.path.join(root, metrics_name)
            metrics = json.load(open(full_name))
            results_dict[full_name] = metrics


    sorted_keys = sorted(list(results_dict.keys()))
    print(f"model_run, {', '.join(keys)}")
    for name in sorted_keys:
        results = results_dict[name]
        keys_to_print = [str(results.get(key, "N/A")) for key in keys]
        print(f"{name}, {', '.join(keys_to_print)}")
1212754
def diff(args):
    
    from jcvi.utils.cbook import SummaryStats

    p = OptionParser(diff.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    simplefile, = args
    fp = open(simplefile)
    data = [x.split() for x in fp]
    spans = []
    for block_id, ab in groupby(data[1:], key=lambda x: x[0]):
        a, b = list(ab)
        aspan, bspan = a[4], b[4]
        aspan, bspan = int(aspan), int(bspan)
        spans.append((aspan, bspan))
    aspans, bspans = zip(*spans)
    dspans = [b - a for a, b, in spans]
    s = SummaryStats(dspans)
    print("For a total of {0} blocks:".format(len(dspans)), file=sys.stderr)
    print("Sum of A: {0}".format(sum(aspans)), file=sys.stderr)
    print("Sum of B: {0}".format(sum(bspans)), file=sys.stderr)
    print("Sum of Delta: {0} ({1})".format(sum(dspans), s), file=sys.stderr)
635109
def stats(args):
    
    from jcvi.utils.cbook import percentage

    p = OptionParser(stats.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    blocksfile, = args
    fp = open(blocksfile)
    counts = defaultdict(int)
    total = orthologous = 0
    for row in fp:
        atoms = row.rstrip().split("\t")
        hits = [x for x in atoms[1:] if x != '.']
        counts[len(hits)] += 1
        total += 1
        if atoms[1] != '.':
            orthologous += 1

    print("Total lines: {0}".format(total), file=sys.stderr)
    for i, n in sorted(counts.items()):
        print("Count {0}: {1}".format(i, percentage(n, total)), file=sys.stderr)

    print(file=sys.stderr)

    matches = sum(n for i, n in counts.items() if i != 0)
    print("Total lines with matches: {0}".\
                format(percentage(matches, total)), file=sys.stderr)
    for i, n in sorted(counts.items()):
        if i == 0:
            continue

        print("Count {0}: {1}".format(i, percentage(n, matches)), file=sys.stderr)

    print(file=sys.stderr)
    print("Orthologous matches: {0}".\
                format(percentage(orthologous, matches)), file=sys.stderr)
230217
def _generate_report(ret, setup):
    '''
    
    '''

    retdata = ret.get('return', {})

    sorted_data = sorted(
        retdata.items(),
        key=lambda s: s[1].get('__run_num__', 0)
    )

    total = 0
    failed = 0
    changed = 0
    duration = 0.0

    # gather stats
    for _, data in sorted_data:
        if not data.get('result', True):
            failed += 1
        total += 1

        try:
            duration += float(data.get('duration', 0.0))
        except ValueError:
            pass

        if data.get('changes', {}):
            changed += 1

    unchanged = total - failed - changed

    log.debug('highstate total: %s', total)
    log.debug('highstate failed: %s', failed)
    log.debug('highstate unchanged: %s', unchanged)
    log.debug('highstate changed: %s', changed)

    # generate report if required
    if setup.get('report_everything', False) or \
       (setup.get('report_changes', True) and changed != 0) or \
       (setup.get('report_failures', True) and failed != 0):

        report = [
            {'stats': [
                {'total': total},
                {'failed': failed, '__style__': 'failed'},
                {'unchanged': unchanged, '__style__': 'unchanged'},
                {'changed': changed, '__style__': 'changed'},
                {'duration': duration}
            ]},
            {'job': [
                {'function': ret.get('fun', '')},
                {'arguments': ret.get('fun_args', '')},
                {'jid': ret.get('jid', '')},
                {'success': ret.get('success', True)},
                {'retcode': ret.get('retcode', 0)}
            ]},
            {'states': _generate_states_report(sorted_data)}
        ]

        if failed:
            function = setup.get('failure_function', None)
        else:
            function = setup.get('success_function', None)

        if function:
            func_result = __salt__[function]()
            report.insert(
                0,
                {'extra': [{function: _dict_to_name_value(func_result)}]}
            )

    else:
        report = []

    return report, failed
607196
def _create_text_report(inputfile, motifs, closest_match, stats, outdir):
    """"""
    my_stats = {}
    for motif in motifs:
        match = closest_match[motif.id]
        my_stats[str(motif)] = {}
        for bg in list(stats.values())[0].keys():
            if str(motif) not in stats:
                logger.error("####")
                logger.error("{} not found".format(str(motif)))
                for s in sorted(stats.keys()):
                    logger.error(s)
                logger.error("####")
            else:
                my_stats[str(motif)][bg] = stats[str(motif)][bg].copy()
                my_stats[str(motif)][bg]["best_match"] = "_".join(match[0].split("_")[:-1])
                my_stats[str(motif)][bg]["best_match_pvalue"] = match[1][-1]
    
    header = ("# GimmeMotifs version {}\n"
             "# Inputfile: {}\n"
             ).format(__version__, inputfile)

    write_stats(my_stats, os.path.join(outdir, "stats.{}.txt"), header=header)
704626
def subset(args):
    
    p = OptionParser(subset.__doc__)
    p.add_option("--noheader", action="store_true",
                 help="don't write ksfile header line [default: %default]")
    p.add_option("--block", action="store_true",
                 help="preserve block structure in input [default: %default]")
    p.set_stripnames()
    p.set_outfile()

    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    pairsfile, ksfiles = args[0], args[1:]
    noheader = opts.noheader
    block = opts.block
    if block:
        noheader = True
    outfile = opts.outfile

    ksvals = {}
    for ksfile in ksfiles:
        ksvals.update(dict((line.name, line) for line in \
                        KsFile(ksfile, strip_names=opts.strip_names)))

    fp = open(pairsfile)
    fw = must_open(outfile, "w")

    if not noheader:
        print(fields, file=fw)

    i = j = 0
    for row in fp:
        if row[0] == '#':
            if block:
                print(row.strip(), file=fw)
            continue
        a, b = row.split()[:2]
        name = ";".join((a, b))
        if name not in ksvals:
            name = ";".join((b, a))
            if name not in ksvals:
                j += 1
                print("\t".join((a, b, ".", ".")), file=fw)
                continue
        ksline = ksvals[name]
        if block:
            print("\t".join(str(x) for x in (a, b, ksline.ks)), file=fw)
        else:
            ksline.name = ";".join((a, b))
            print(ksline, file=fw)
        i += 1
    fw.close()

    logging.debug("{0} pairs not found in ksfiles".format(j))
    logging.debug("{0} ks records written to `{1}`".format(i, outfile))
    return outfile
1244672
def roc(args):
    
    outputfile = args.outfile
    # Default extension for image
    if outputfile and not outputfile.endswith(".png"):
        outputfile += ".png"
    
    motifs = read_motifs(args.pwmfile, fmt="pwm")

    ids = []
    if args.ids:
        ids = args.ids.split(",")
    else:
        ids = [m.id for m in motifs]
    motifs = [m for m in motifs if (m.id in ids)]
    
    stats = [
            "phyper_at_fpr",
            "roc_auc", 
            "pr_auc", 
            "enr_at_fpr",
            "recall_at_fdr", 
            "roc_values",
            "matches_at_fpr",
            ]
    
    plot_x = []
    plot_y = []
    legend = []
    
    f_out = sys.stdout
    if args.outdir:
        if not os.path.exists(args.outdir):
            os.makedirs(args.outdir)
        f_out = open(args.outdir + "/gimme.roc.report.txt", "w")
    
    # Print the metrics
    f_out.write("Motif\t# matches\t# matches background\tP-value\tlog10 P-value\tROC AUC\tPR AUC\tEnr. at 1% FPR\tRecall at 10% FDR\n")
    
    
    for motif_stats in calc_stats_iterator(motifs, args.sample, args.background, 
            genome=args.genome, stats=stats, ncpus=args.ncpus):
    
        for motif in motifs:
            if str(motif) in motif_stats:
                if outputfile:
                    x, y = motif_stats[str(motif)]["roc_values"]
                    plot_x.append(x)
                    plot_y.append(y)
                    legend.append(motif.id)
                log_pvalue = np.inf
                if motif_stats[str(motif)]["phyper_at_fpr"] > 0:
                    log_pvalue = -np.log10(motif_stats[str(motif)]["phyper_at_fpr"])
                f_out.write("{}\t{:d}\t{:d}\t{:.2e}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.2f}\t{:0.4f}\n".format(
                      motif.id, 
                      motif_stats[str(motif)]["matches_at_fpr"][0], 
                      motif_stats[str(motif)]["matches_at_fpr"][1], 
                      motif_stats[str(motif)]["phyper_at_fpr"], 
                      log_pvalue, 
                      motif_stats[str(motif)]["roc_auc"], 
                      motif_stats[str(motif)]["pr_auc"], 
                      motif_stats[str(motif)]["enr_at_fpr"], 
                      motif_stats[str(motif)]["recall_at_fdr"],
                      ))
    f_out.close() 
    
    if args.outdir:
        html_report(
            args.outdir,
            args.outdir + "/gimme.roc.report.txt",
            args.pwmfile,
            0.01,
            )

    # Plot the ROC curve
    if outputfile:
        roc_plot(outputfile, plot_x, plot_y, ids=legend)
783759
def create_report(outdirname, report_filename, **kwargs):
    
    # Checking the required variables
    if "steps" in kwargs:
        assert "descriptions" in kwargs
        assert "long_descriptions" in kwargs
        assert "steps_filename" not in kwargs
    else:
        assert "steps_filename" in kwargs
        assert "descriptions" not in kwargs
        assert "long_descriptions" not in kwargs
    assert "summaries" in kwargs
    assert "background" in kwargs
    assert "project_name" in kwargs
    assert "summary_fn" in kwargs
    assert "report_title" in kwargs
    assert "report_author" in kwargs
    assert "initial_files" in kwargs
    assert "final_nb_markers" in kwargs
    assert "final_nb_samples" in kwargs
    assert "final_files" in kwargs
    assert "plink_version" in kwargs
    assert "graphic_paths_fn" in kwargs

    # Formatting the background section
    background_section = _format_background(kwargs["background"])

    # Writing the method steps to a separate file (for access later)
    steps_filename = None
    if "steps_filename" in kwargs:
        steps_filename = kwargs["steps_filename"]
    else:
        steps_filename = os.path.join(outdirname, "steps_summary.tex")
        with open(steps_filename, "w") as o_file:
            zipped = zip(kwargs["steps"], kwargs["descriptions"],
                         kwargs["long_descriptions"])
            for step, desc, long_desc in zipped:
                if desc.endswith("."):
                    desc = desc[:-1]
                step = step.replace("_", r"\_")
                to_print = latex.item(desc)
                to_print += " [{}].".format(latex.texttt(step))
                if long_desc is not None:
                    to_print += " " + long_desc
                print >>o_file, latex.wrap_lines(to_print) + "\n"

    # Adding the content of the results section
    result_summaries = []
    for name in kwargs["summaries"]:
        full_path = os.path.abspath(name)
        if os.path.isfile(full_path):
            rel_path = os.path.relpath(full_path, outdirname)
            result_summaries.append(re.sub(r"\\", "/", rel_path))

    # Reading the initial_files file
    initial_files = None
    with open(kwargs["initial_files"], "r") as i_file:
        initial_files = i_file.read().splitlines()

    # Reading the final_files file
    final_files = None
    with open(kwargs["final_files"], "r") as i_file:
        final_files = [i.split("\t")[0] for i in i_file.read().splitlines()]

    # Adding the bibliography content
    biblio_entry = latex.bib_entry(
        name="pyGenClean",
        authors="Lemieux Perreault LP, Provost S, Legault MA, Barhdadi A, "
                r"Dub\'e MP",
        title="pyGenClean: efficient tool for genetic data clean up before "
              "association testing",
        journal="Bioinformatics",
        year="2013",
        volume="29",
        number="13",
        pages="1704--1705",
    ) + "\n" * 2 + latex.bib_entry(
        name="plink",
        authors="Purcell S, Neale B, Todd-Brown K, Thomas L, Ferreira MAR, "
                "Bender D, Maller J, Sklar P, de Bakker PIW, Daly MJ, Sham PC",
        title="PLINK: a tool set for whole-genome association and "
              "population-based linkage analyses",
        journal="American Journal of Human Genetics",
        year="2007",
        volume="81",
        number="3",
        pages="559--575",
    ) + "\n" * 2 + latex.bib_entry(
        name="bafRegress",
        authors=r"Goo J, Matthew F, Kurt NH, Jane MR, Kimberly FD, "
                r"Gon{\c{c}}alo RA, Michael B, Hyun Min K",
        title="Detecting and estimating contamination of human DNA samples in "
              "sequencing and array-based genotype data",
        journal="The American Journal of Human Genetics",
        year="2012",
        volume="91",
        number="5",
        pages="839--848",
    )

    # Getting the template
    main_template = latex.jinja2_env.get_template("main_document.tex")

    # Getting the data
    today = datetime.today()

    # Reading the graphics path
    graphic_paths = []
    if kwargs["graphic_paths_fn"] is not None:
        with open(kwargs["graphic_paths_fn"], "r") as i_file:
            graphic_paths = [
                re.sub(r"\\", "/", path) + ("" if path.endswith("/") else "/")
                for path in i_file.read().splitlines()
            ]

    try:
        with open(report_filename, "w") as i_file:
            # Rendering the template
            print >>i_file, main_template.render(
                project_name=latex.sanitize_tex(kwargs["project_name"]),
                month=today.strftime("%B"),
                day=today.day,
                year=today.year,
                background_content=background_section,
                result_summaries=result_summaries,
                bibliography_content=biblio_entry,
                pygenclean_version=pygenclean_version,
                plink_version=kwargs["plink_version"],
                steps_filename=os.path.basename(steps_filename),
                final_results=_create_summary_table(
                    kwargs["summary_fn"],
                    latex.jinja2_env.get_template("summary_table.tex"),
                    nb_samples=kwargs["final_nb_samples"],
                    nb_markers=kwargs["final_nb_markers"],
                ),
                report_title=latex.sanitize_tex(kwargs["report_title"]),
                report_author=latex.sanitize_tex(kwargs["report_author"]),
                initial_files=initial_files,
                final_files=final_files,
                final_nb_samples=kwargs["final_nb_samples"],
                final_nb_markers=kwargs["final_nb_markers"],
                graphic_paths=graphic_paths,
            )

    except IOError:
        msg = "{}: could not create report".format(report_filename)
        raise ProgramError(msg)
366101
def print_results(distributions, list_files=False, verbose=False):
    
    results_printed = False
    for i, dist in enumerate(distributions):
        results_printed = True
        if i > 0:
            logger.info("---")

        name = dist.get('name', '')
        required_by = [
            pkg.project_name for pkg in pkg_resources.working_set
            if name in [required.name for required in pkg.requires()]
        ]

        logger.info("Name: %s", name)
        logger.info("Version: %s", dist.get('version', ''))
        logger.info("Summary: %s", dist.get('summary', ''))
        logger.info("Home-page: %s", dist.get('home-page', ''))
        logger.info("Author: %s", dist.get('author', ''))
        logger.info("Author-email: %s", dist.get('author-email', ''))
        logger.info("License: %s", dist.get('license', ''))
        logger.info("Location: %s", dist.get('location', ''))
        logger.info("Requires: %s", ', '.join(dist.get('requires', [])))
        logger.info("Required-by: %s", ', '.join(required_by))

        if verbose:
            logger.info("Metadata-Version: %s",
                        dist.get('metadata-version', ''))
            logger.info("Installer: %s", dist.get('installer', ''))
            logger.info("Classifiers:")
            for classifier in dist.get('classifiers', []):
                logger.info("  %s", classifier)
            logger.info("Entry-points:")
            for entry in dist.get('entry_points', []):
                logger.info("  %s", entry.strip())
        if list_files:
            logger.info("Files:")
            for line in dist.get('files', []):
                logger.info("  %s", line.strip())
            if "files" not in dist:
                logger.info("Cannot locate installed-files.txt")
    return results_printed
903202
def output(results_dir, config, parent='../../'):
    
    start = time.time()
    print("Compiling results...")
    results_dir = os.path.abspath(results_dir)
    results = ReportResults(config['run_time'], config['results_ts_interval'])
    results.compile_results()
    print("Results compiled in {} seconds\n".format(time.time() - start))

    if results.total_transactions == 0:
        print("No results, cannot create report")
        return False

    print_infos(results)

    data = {
        'report': results,
        'run_time': config['run_time'],
        'ts_interval': config['results_ts_interval'],
        'turrets_config': results.turrets,
        'results': {"all": results.main_results, "timers": results.timers_results}
    }

    print("Generating graphs...")
    partial = time.time()
    generate_graphs(results.main_results, 'All_Transactions', results_dir)

    for key, value in results.timers_results.items():
        generate_graphs(value, key, results_dir)
    print("All graphs generated in {} seconds\n".format(time.time() - partial))

    write_template(data, results_dir, parent)
    print("Full report generated in {} seconds".format(time.time() - start))
    return True
221058
def prt_summary(self, prt=sys.stdout):
        """"""
        desc = "NtGoeaResults" if self.is_goterm else "namedtuple"
        prt.write("{N} GOEA results from {O}. P-values stored in {P}.\n".format(
            N=len(self.go2res), O=desc, P=self.pval_name))
1144678
def make_report(self, outcome):
        
        failures = self.getreports('failed')
        if not failures:
            return
        for rep in failures:
            # Check if this is a notebook node
            msg = self._getfailureheadline(rep)
            lines = rep.longrepr.splitlines()
            if len(lines) > 1:
                self.section(msg, lines[1])
            self._outrep_summary(rep)
        tmpdir = tempfile.mkdtemp()
        try:
            ref_file = os.path.join(tmpdir, 'reference.ipynb')
            test_file = os.path.join(tmpdir, 'test_result.ipynb')
            with io.open(ref_file, "w", encoding="utf8") as f:
                nbformat.write(self.nb_ref, f)
            with io.open(test_file, "w", encoding="utf8") as f:
                nbformat.write(self.nb_test, f)
            run_server(
                port=0,     # Run on random port
                cwd=tmpdir,
                closable=True,
                on_port=lambda port: browse(
                    port, ref_file, test_file, None))
        finally:
            shutil.rmtree(tmpdir)
400817
def report(options)
      report = HamlLint::Report.new(reporter: options[:reporter], fail_level: options[:fail_level])
      report.start(@files)
      process_files(report)
      report
    end
1165527
def write_diff_summary(stats, options):
    
    # set output file name
    out_name = '%s_diff_stats.csv' % options.outname
    filename = os.path.join(os.getcwd(), out_name)

    # open file & create csv writer object
    f = open(filename, 'w')
    statswriter = csv.writer(f)

    # write header
    statswriter.writerow(stats['header'])

    # write auc
    line = ['AUC']
    for metric_tup in stats['AUC']:
        try:
            len(metric_tup)
            line.append('[%0.2f, %0.2f]' % (metric_tup[0], metric_tup[1]))
        except:
            line.append('%0.6f' % metric_tup)
    statswriter.writerow(line)

    # write enrichment factors
    for title in [x for x in sorted(stats.keys()) if 'E' in x]:
        line = [title]
        for metric_tup in stats[title]:
            try:
                len(metric_tup)
                line.append('[%0.2f, %0.2f]' % (metric_tup[0], metric_tup[1]))
            except:
                line.append('%0.12f' % metric_tup)
        statswriter.writerow(line)
49413
def summary(args):
    
    from jcvi.formats.blast import AlignStats

    p = OptionParser(summary.__doc__)
    p.add_option("-s", dest="single", default=False, action="store_true",
            help="provide stats per reference seq")

    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(p.print_help())

    coordsfile, = args
    alignstats = get_stats(coordsfile)
    alignstats.print_stats()