take the zscore of a dataframe dictionary, does not write to net (self)
342766
def zscore_df(df, axis='row', keep_orig=False):
  '''
  
  '''
  df_z = {}

  for mat_type in df:
    if keep_orig and mat_type == 'mat':
      mat_orig = deepcopy(df[mat_type])

    inst_df = df[mat_type]

    if axis == 'row':
      inst_df = inst_df.transpose()

    df_z[mat_type] = (inst_df - inst_df.mean())/inst_df.std()

    if axis == 'row':
      df_z[mat_type] = df_z[mat_type].transpose()

  if keep_orig:
    df_z['mat_orig'] = mat_orig

  return df_z
545544
def zscore(self, name, value):
        
        with self.pipe as pipe:
            return pipe.zscore(self.redis_key(name),
                               self.valueparse.encode(value))
830438
def to_df(self):
    '''
    '''
    total, first, (idx, second) = Si_to_pandas_dict(self)
    names = self.problem['names']
    ret = [pd.DataFrame(total, index=names),
           pd.DataFrame(first, index=names)]

    if second:
        ret += [pd.DataFrame(second, index=idx)]

    return ret
684825
def from_dict(cls, d):
        """"""
        stats = []
        for (filename, lineno, name), stat_values in d.iteritems():
            if len(stat_values) == 5:
                ncalls, ncall_nr, total_time, cum_time, subcall_stats = stat_values
            else:
                ncalls, ncall_nr, total_time, cum_time = stat_values
                subcall_stats = None
            stat = cProfileFuncStat(filename, lineno, name, ncalls, ncall_nr, total_time, cum_time, subcall_stats)
            stats.append(stat)

        return stats
66173
def parse(resp) -> DataFrameType:
    """"""
    statements = []
    for statement in resp['results']:
        series = {}
        for s in statement.get('series', []):
            series[_get_name(s)] = _drop_zero_index(_serializer(s))
        statements.append(series)

    if len(statements) == 1:
        series: dict = statements[0]
        if len(series) == 1:
            return list(series.values())[0]  # DataFrame
        else:
            return series  # dict
    return statements
1166765
def read(self, dataframe_name: str) -> pandas.DataFrame:
        
        code = serialise_dataframe_code(dataframe_name, self.kind)
        output = self._execute(code)
        output.raise_for_status()
        if output.text is None:
            raise RuntimeError("statement had no text output")
        return deserialise_dataframe(output.text)
1213389
def edges_to_dict_of_dataframes(grid, edges):
    
    omega = 2 * pi * 50
    srid = int(cfg_ding0.get('geo', 'srid'))

    lines = {'line_id': [], 'bus0': [], 'bus1': [], 'x': [], 'r': [],
             's_nom': [], 'length': [], 'cables': [], 'geom': [],
             'grid_id': []}

    # iterate over edges and add them one by one
    for edge in edges:

        line_name = '_'.join(['MV',
                              str(grid.id_db),
                              'lin',
                              str(edge['branch'].id_db)])

        # TODO: find the real cause for being L, C, I_th_max type of Series
        if (isinstance(edge['branch'].type['L'], Series) or
                isinstance(edge['branch'].type['C'], Series)):
            x = omega * edge['branch'].type['L'].values[0] * 1e-3
        else:

            x = omega * edge['branch'].type['L'] * 1e-3

        if isinstance(edge['branch'].type['R'], Series):
            r = edge['branch'].type['R'].values[0]
        else:
            r = edge['branch'].type['R']

        if (isinstance(edge['branch'].type['I_max_th'], Series) or
                isinstance(edge['branch'].type['U_n'], Series)):
            s_nom = sqrt(3) * edge['branch'].type['I_max_th'].values[0] * \
                    edge['branch'].type['U_n'].values[0]
        else:
            s_nom = sqrt(3) * edge['branch'].type['I_max_th'] * \
                    edge['branch'].type['U_n']

        # get lengths of line
        l = edge['branch'].length / 1e3

        lines['line_id'].append(line_name)
        lines['bus0'].append(edge['adj_nodes'][0].pypsa_id)
        lines['bus1'].append(edge['adj_nodes'][1].pypsa_id)
        lines['x'].append(x * l)
        lines['r'].append(r * l)
        lines['s_nom'].append(s_nom)
        lines['length'].append(l)
        lines['cables'].append(3)
        lines['geom'].append(from_shape(
            LineString([edge['adj_nodes'][0].geo_data,
                        edge['adj_nodes'][1].geo_data]),
            srid=srid))
        lines['grid_id'].append(grid.id_db)

    return {'Line': DataFrame(lines).set_index('line_id')}
1063866
def df_filter_col_sum(df, threshold, take_abs=True):
  '''  '''

  from copy import deepcopy
  from .__init__ import Network
  net = Network()

  if take_abs is True:
    df_copy = deepcopy(df['mat'].abs())
  else:
    df_copy = deepcopy(df['mat'])

  df_copy = df_copy.transpose()
  df_copy = df_copy[df_copy.sum(axis=1) > threshold]
  df_copy = df_copy.transpose()
  df_copy = df_copy[df_copy.sum(axis=1) > 0]

  if take_abs is True:
    inst_rows = df_copy.index.tolist()
    inst_cols = df_copy.columns.tolist()
    df['mat'] = grab_df_subset(df['mat'], inst_rows, inst_cols)

    if 'mat_up' in df:
      df['mat_up'] = grab_df_subset(df['mat_up'], inst_rows, inst_cols)
      df['mat_dn'] = grab_df_subset(df['mat_dn'], inst_rows, inst_cols)

    if 'mat_orig' in df:
      df['mat_orig'] = grab_df_subset(df['mat_orig'], inst_rows, inst_cols)

  else:
    df['mat'] = df_copy

  return df
1262021
def to_dict(self, columns=None):
        
        if columns:
            assert len(columns) == len(self.feature_names)
        d = [
            dict([
                (columns[i] if columns else col_name, row[i])
                for i, col_name in enumerate(self.feature_names)
            ])
            for row in self.data
        ]
        return d
1043742
def write_dataframe(self, result, dst_paths, nodata=None, compress='lzw'):
        """"""
        result = self._convert_to_ndarray(result)
        self.write_ndarray(result, dst_paths, nodata=nodata, compress=compress)
374597
def _remove_nonascii(self, df):
    """"""

    df_copy = df.copy(deep=True)
    for col in df_copy.columns:
      if (df_copy[col].dtype == np.dtype('O')):
        df_copy[col] = df[col].apply(
          lambda x: re.sub(r'[^\x00-\x7f]', r'', x) if isinstance(x, six.string_types) else x)

    return df_copy
1329667
def clean_dict(d, test=lambda v: v):
    
    return {k: v for k, v in d.items() if test(v)}
332273
def df_filter_row_sum(df, threshold, take_abs=True):
  '''  '''

  from copy import deepcopy
  from .__init__ import Network
  net = Network()

  if take_abs is True:
    df_copy = deepcopy(df['mat'].abs())
  else:
    df_copy = deepcopy(df['mat'])

  ini_rows = df_copy.index.values.tolist()
  df_copy = df_copy.transpose()
  tmp_sum = df_copy.sum(axis=0)
  tmp_sum = tmp_sum.abs()
  tmp_sum.sort_values(inplace=True, ascending=False)

  tmp_sum = tmp_sum[tmp_sum > threshold]
  keep_rows = sorted(tmp_sum.index.values.tolist())

  if len(keep_rows) < len(ini_rows):
    df['mat'] = grab_df_subset(df['mat'], keep_rows=keep_rows)

    if 'mat_up' in df:
      df['mat_up'] = grab_df_subset(df['mat_up'], keep_rows=keep_rows)
      df['mat_dn'] = grab_df_subset(df['mat_dn'], keep_rows=keep_rows)

    if 'mat_orig' in df:
      df['mat_orig'] = grab_df_subset(df['mat_orig'], keep_rows=keep_rows)

  return df
1250020
def _get_s(cls, df, column, shifts):
        
        shift = cls.to_int(shifts)
        shifted_key = "{}_{}_s".format(column, shift)
        df[shifted_key] = df[column].shift(-shift)
        StockDataFrame.set_nan(df[shifted_key], shift)
1306857
def _get_bin(self, key):
        '''
        
        '''
        # keys based on score
        sortedDict = {}
        # this doesnt return them in order, need to bin first
        for item in self.redis_conn.zscan_iter(key):
            my_item = ujson.loads(item[0])
            # score is negated in redis
            my_score = -item[1]

            if my_score not in sortedDict:
                sortedDict[my_score] = []

            sortedDict[my_score].append(my_item)

        return sortedDict
273843
def format_stats(self, stats:TensorOrNumList)->None:
        ""
        str_stats = []
        for name,stat in zip(self.names,stats):
            str_stats.append('#na#' if stat is None else str(stat) if isinstance(stat, int) else f'{stat:.6f}')
        if self.add_time: str_stats.append(format_time(time() - self.start_epoch))
        if not self.silent: self.pbar.write(str_stats, table=True)
1058878
def dump_dict(self):
        """"""

        dump_dict = dict()

        dump_dict['Structure'] = self.name

        # Refer to the __set_format__ method for an explanation
        # of the following construct.
        for keys in self.__keys__:
            for key in keys:

                val = getattr(self, key)
                if isinstance(val, (int, long)):
                    if key == 'TimeDateStamp' or key == 'dwTimeStamp':
                        try:
                            val = '0x%-8X [%s UTC]' % (val, time.asctime(time.gmtime(val)))
                        except ValueError as e:
                            val = '0x%-8X [INVALID TIME]' % val
                else:
                    val = ''.join(chr(d) if chr(d) in string.printable
                                  else "\\x%02x" % d for d in
                                    [ord(c) if not isinstance(c, int) else c for c in val])

                dump_dict[key] = {'FileOffset': self.__field_offsets__[key] + self.__file_offset__,
                                  'Offset': self.__field_offsets__[key],
                                  'Value': val}

        return dump_dict