Combines calculate_metrics, write_dataframe_to_csv, and plot
172195
def full_analysis(self, analysis_set, output_directory, verbose = True, compile_pdf = True, quick_plots = False):
        ''''''
        if not os.path.isdir(output_directory):
            os.makedirs(output_directory)
        self.analysis_directory = output_directory
        self.calculate_metrics(analysis_set = analysis_set, analysis_directory = output_directory, verbose = verbose)
        self.write_dataframe_to_csv( os.path.join(output_directory, 'data.csv') )

        # Return latex_report
        return self.plot(analysis_set = analysis_set, analysis_directory = output_directory, matplotlib_plots = True, verbose = verbose, compile_pdf = compile_pdf, quick_plots = quick_plots)
316389
def _apply_summaries(self):
        """"""

        def as_frame(r):
            if isinstance(r, pd.Series):
                return r.to_frame()
            else:
                return r

        df = self.data

        if df.index.nlevels > 1:
            raise ValueError(
                "You cannot currently have both summary rows and columns on a "
                "MultiIndex."
            )

        _df = df
        if self.summary_rows:
            rows = pd.concat([agg.apply(_df)
                              for agg in self._cleaned_summary_rows], axis=1).T
            df = pd.concat([df, as_frame(rows)], axis=0)

        if self.summary_cols:
            cols = pd.concat([agg.apply(_df)
                              for agg in self._cleaned_summary_cols], axis=1)
            df = pd.concat([df, as_frame(cols)], axis=1)

        return df
856253
def write_krona_plot(self, sample_names, read_taxonomies, output_krona_filename):
        ''''''
        tempfiles = []
        for n in sample_names:
            tempfiles.append(tempfile.NamedTemporaryFile(prefix='GraftMkronaInput', suffix=n))
        
        delim=u'\t'
        for _, tax, counts in self._iterate_otu_table_rows(read_taxonomies):
            for i, c in enumerate(counts):
                if c != 0:
                    tempfiles[i].write(delim.join((str(c),
                                                  delim.join(tax)
                                                  ))+"\n")
                    
        for t in tempfiles:
            t.flush()
        
        cmd = ["ktImportText",'-o',output_krona_filename]
        for i, tmp in enumerate(tempfiles):
            cmd.append(','.join([tmp.name,sample_names[i]]))

        # run the actual krona
        cmd = ' '.join(cmd)
        extern.run(cmd)

        # close tempfiles
        for t in tempfiles:
            t.close()
1208339
def write_metrics_file(metrics: List[Dict[str, Any]], path: str):
    
    with open(path, 'w') as metrics_out:
        for checkpoint, metric_dict in enumerate(metrics, 1):
            metrics_str = "\t".join(["{}={}".format(name, value) for name, value in sorted(metric_dict.items())])
            metrics_out.write("{}\t{}\n".format(checkpoint, metrics_str))
991501
def _write_csv(self, df, csv_path, chunksize=10**5):
        
        logger.info("Saving DataFrame to %s", csv_path)
        df.to_csv(csv_path, index=False, chunksize=chunksize)
1165527
def write_diff_summary(stats, options):
    
    # set output file name
    out_name = '%s_diff_stats.csv' % options.outname
    filename = os.path.join(os.getcwd(), out_name)

    # open file & create csv writer object
    f = open(filename, 'w')
    statswriter = csv.writer(f)

    # write header
    statswriter.writerow(stats['header'])

    # write auc
    line = ['AUC']
    for metric_tup in stats['AUC']:
        try:
            len(metric_tup)
            line.append('[%0.2f, %0.2f]' % (metric_tup[0], metric_tup[1]))
        except:
            line.append('%0.6f' % metric_tup)
    statswriter.writerow(line)

    # write enrichment factors
    for title in [x for x in sorted(stats.keys()) if 'E' in x]:
        line = [title]
        for metric_tup in stats[title]:
            try:
                len(metric_tup)
                line.append('[%0.2f, %0.2f]' % (metric_tup[0], metric_tup[1]))
            except:
                line.append('%0.12f' % metric_tup)
        statswriter.writerow(line)
159777
def write(self, handle):
        '''
        '''
        if not self._frames:
            return

        def add(name, desc, bpe, format, bytes, *dimensions):
            group.add_param(name,
                            desc=desc,
                            bytes_per_element=bpe,
                            bytes=struct.pack(format, bytes),
                            dimensions=list(dimensions))

        def add_str(name, desc, bytes, *dimensions):
            group.add_param(name,
                            desc=desc,
                            bytes_per_element=-1,
                            bytes=bytes.encode('utf-8'),
                            dimensions=list(dimensions))

        def add_empty_array(name, desc, bpe):
            group.add_param(name, desc=desc, bytes_per_element=bpe, dimensions=[0])

        points, analog = self._frames[0]
        ppf = len(points)

        # POINT group
        group = self.add_group(1, 'POINT', 'POINT group')
        add('USED', 'Number of 3d markers', 2, '<H', ppf)
        add('FRAMES', 'frame count', 2, '<H', min(65535, len(self._frames)))
        add('DATA_START', 'data block number', 2, '<H', 0)
        add('SCALE', '3d scale factor', 4, '<f', self._point_scale)
        add('RATE', '3d data capture rate', 4, '<f', self._point_rate)
        add_str('X_SCREEN', 'X_SCREEN parameter', '+X', 2)
        add_str('Y_SCREEN', 'Y_SCREEN parameter', '+Y', 2)
        add_str('UNITS', '3d data units', self._point_units, len(self._point_units))
        add_str('LABELS', 'labels', ''.join('M%03d ' % i for i in range(ppf)), 5, ppf)
        add_str('DESCRIPTIONS', 'descriptions', ' ' * 16 * ppf, 16, ppf)

        # ANALOG group
        group = self.add_group(2, 'ANALOG', 'ANALOG group')
        add('USED', 'analog channel count', 2, '<H', analog.shape[0])
        add('RATE', 'analog samples per 3d frame', 4, '<f', analog.shape[1])
        add('GEN_SCALE', 'analog general scale factor', 4, '<f', self._gen_scale)
        add_empty_array('SCALE', 'analog channel scale factors', 4)
        add_empty_array('OFFSET', 'analog channel offsets', 2)

        # TRIAL group
        group = self.add_group(3, 'TRIAL', 'TRIAL group')
        add('ACTUAL_START_FIELD', 'actual start frame', 2, '<I', 1, 2)
        add('ACTUAL_END_FIELD', 'actual end frame', 2, '<I', len(self._frames), 2)

        # sync parameter information to header.
        blocks = self.parameter_blocks()
        self.get('POINT:DATA_START').bytes = struct.pack('<H', 2 + blocks)

        self.header.data_block = 2 + blocks
        self.header.frame_rate = self._point_rate
        self.header.last_frame = min(len(self._frames), 65535)
        self.header.point_count = ppf
        self.header.analog_count = np.prod(analog.shape)
        self.header.analog_per_frame = analog.shape[0]
        self.header.scale_factor = self._point_scale

        self._write_metadata(handle)
        self._write_frames(handle)
273843
def format_stats(self, stats:TensorOrNumList)->None:
        ""
        str_stats = []
        for name,stat in zip(self.names,stats):
            str_stats.append('#na#' if stat is None else str(stat) if isinstance(stat, int) else f'{stat:.6f}')
        if self.add_time: str_stats.append(format_time(time() - self.start_epoch))
        if not self.silent: self.pbar.write(str_stats, table=True)
822718
def Main():
  
  argument_parser = argparse.ArgumentParser(description=(
      'Plots memory usage from profiling data.'))

  argument_parser.add_argument(
      '--output', dest='output_file', type=str, help=(
          'path of the output file to write the graph to instead of using '
          'interactive mode. The output format deduced from the extension '
          'of the filename.'))

  argument_parser.add_argument(
      'profile_path', type=str, help=(
          'path to the directory containing the profiling data.'))

  options = argument_parser.parse_args()

  if not os.path.isdir(options.profile_path):
    print('No such directory: {0:s}'.format(options.profile_path))
    return False

  names = ['time', 'queued', 'processing', 'to_merge', 'abandoned', 'total']

  glob_expression = os.path.join(options.profile_path, 'task_queue-*.csv.gz')
  for csv_file_name in glob.glob(glob_expression):
    data = numpy.genfromtxt(
        csv_file_name, delimiter='\t', dtype=None, encoding='utf-8',
        names=names, skip_header=1)

    pyplot.plot(data['time'], data['queued'], label='queued')
    pyplot.plot(data['time'], data['processing'], label='processing')
    pyplot.plot(data['time'], data['to_merge'], label='to merge')
    pyplot.plot(data['time'], data['abandoned'], label='abandoned')

  pyplot.title('Number of tasks over time')

  pyplot.xlabel('Time')
  pyplot.xscale('linear')

  pyplot.ylabel('Number of tasks')
  pyplot.yscale('linear')

  pyplot.legend()

  if options.output_file:
    pyplot.savefig(options.output_file)
  else:
    pyplot.show()

  return True
185123
def generate_pmlb_summary(local_cache_dir=None):
    
    report_filename = open(os.path.join(local_cache_dir, 'report.csv'), 'wt')
    assert (local_cache_dir!=None)
    try:
        writer = csv.writer(report_filename, delimiter='\t')
        writer.writerow(['Dataset','#instances','#features','#binary_features','#integer_features','#float_features',\
                     'Endpoint_type','#classes','Imbalance_metric'])

        for dataset in dataset_names:
            df=fetch_data(dataset)
            print( "Dataset:", dataset)
            assert 'class' in df.columns, "no class column"
            #removing class column
            print( "SIZE: "+ str(len(df.axes[0]))+ " x " + str(len(df.axes[1])-1))
            feat=count_features_type(df.ix[:, df.columns != 'class'])
            endpoint=determine_endpoint_type(df.ix[:, df.columns == 'class'])
            mse=imbalance_metrics(df['class'].tolist())
            #writer.writerow([file,str(len(df.axes[0])),str(len(df.axes[1])-1),feat[0],feat[1],feat[2],endpoint,mse[0],mse[1],mse[2]])
            writer.writerow([dataset,str(len(df.axes[0])),str(len(df.axes[1])-1),feat[0],feat[1],feat[2],endpoint,int(mse[0]),mse[1]])
    finally:
        report_filename.close()
538119
def stats(self, columns):
        
        def reduce_func(sc1, sc2):
            return sc1.merge_pstats(sc2)

        return self._rdd.mapPartitions(lambda partition: [
            PStatCounter(dataframes=partition, columns=columns)])\
            .reduce(reduce_func)
751134
def log_metrics(metrics, summ_writer, log_prefix, step, history=None):
  """"""
  rjust_len = max([len(name) for name in metrics])
  for name, value in six.iteritems(metrics):
    step_log(step, "%s %s | % .8f" % (
        log_prefix.ljust(5), name.rjust(rjust_len), value))
    full_name = "metrics/" + name
    if history:
      history.append(log_prefix, full_name, step, value)
    if summ_writer:
      summ_writer.scalar(full_name, value, step)
1145884
def write_csvs(dirname: PathLike, adata: AnnData, skip_data: bool = True, sep: str = ','):
    
    dirname = Path(dirname)
    if dirname.suffix == '.csv':
        dirname = dirname.with_suffix('')
    logger.info("writing '.csv' files to %s", dirname)
    if not dirname.is_dir():
        dirname.mkdir(parents=True, exist_ok=True)
    dir_uns = dirname / 'uns'
    if not dir_uns.is_dir():
        dir_uns.mkdir(parents=True, exist_ok=True)
    d = dict(
        obs=adata._obs,
        var=adata._var,
        obsm=adata._obsm.to_df(),
        varm=adata._varm.to_df(),
    )
    if not skip_data:
        d['X'] = pd.DataFrame(
            adata._X.toarray() if issparse(adata._X) else adata._X)
    d_write = {**d, **adata._uns}
    not_yet_raised_sparse_warning = True
    for key, value in d_write.items():
        if issparse(value):
            if not_yet_raised_sparse_warning:
                warnings.warn('Omitting to write sparse annotation.')
                not_yet_raised_sparse_warning = False
            continue
        filename = dirname
        if key not in {'X', 'var', 'obs', 'obsm', 'varm'}:
            filename = dir_uns
        filename /= '{}.csv'.format(key)
        df = value
        if not isinstance(value, pd.DataFrame):
            value = np.array(value)
            if np.ndim(value) == 0:
                value = value[None]
            try:
                df = pd.DataFrame(value)
            except Exception as e:
                warnings.warn('Omitting to write {!r}.'.format(key), type(e))
                continue
        df.to_csv(
            filename, sep=sep,
            header=key in {'obs', 'var', 'obsm', 'varm'},
            index=key in {'obs', 'var'},
        )
1308434
def output_barplot(df, figformat, path, title=None, palette=None):
    """"""
    logging.info("Nanoplotter: Creating barplots for number of reads and total throughput.")
    read_count = Plot(path=path + "NanoComp_number_of_reads." + figformat,
                      title="Comparing number of reads")
    ax = sns.countplot(x="dataset",
                       data=df,
                       palette=palette)
    ax.set(ylabel='Number of reads',
           title=title or read_count.title)
    plt.xticks(rotation=30, ha='center')
    read_count.fig = ax.get_figure()
    read_count.save(format=figformat)
    plt.close("all")

    throughput_bases = Plot(path=path + "NanoComp_total_throughput." + figformat,
                            title="Comparing throughput in gigabases")
    if "aligned_lengths" in df:
        throughput = df.groupby('dataset')['aligned_lengths'].sum()
        ylabel = 'Total gigabase aligned'
    else:
        throughput = df.groupby('dataset')['lengths'].sum()
        ylabel = 'Total gigabase sequenced'
    ax = sns.barplot(x=list(throughput.index),
                     y=throughput / 1e9,
                     palette=palette,
                     order=df["dataset"].unique())
    ax.set(ylabel=ylabel,
           title=title or throughput_bases.title)
    plt.xticks(rotation=30, ha='center')
    throughput_bases.fig = ax.get_figure()
    throughput_bases.save(format=figformat)
    plt.close("all")
    return read_count, throughput_bases
1132508
function displayMetrics(file) {
    var prefix = '';

    var code = fs.readFileSync(file, 'utf-8');

    // We remove any shebang at the start of the file since that isn't valid
    // Javascript and will trip up the parser.

    code = code.replace(/^#!.*\n/, '');

    var metrics = complexity(code);

    // Walk the metrics structure and add each member to the table rows.

    function walk(data, name) {
        // Fade out anonymous functions.

        if (name.indexOf('anon@') === 0) {
            name = name.grey;
        }

        rows.push([ prefix + name, data.ecc, data.arity, data.codeLines, data.commentLines, Math.round(100 * data.commentLines / data.codeLines) ]);

        // Add two spaces to the prefix before the next depth, to illustrate
        // the hierarchy in the table.

        prefix += '  ';
        _.each(data.children, walk);
        prefix = prefix.slice(0,prefix.length-2);
    }
    walk(metrics, path.basename(file).blue.bold);
}
1043742
def write_dataframe(self, result, dst_paths, nodata=None, compress='lzw'):
        """"""
        result = self._convert_to_ndarray(result)
        self.write_ndarray(result, dst_paths, nodata=nodata, compress=compress)
504630
def plot_memory(calc_id=-1):
    
    dstore = util.read(calc_id)
    plots = []
    for task_name in dstore['task_info']:
        mem = dstore['task_info/' + task_name]['mem_gb']
        plots.append((task_name, mem))
    plt = make_figure(plots)
    plt.show()