Converting metrics to numeric when pandas.read_sql cannot
345342
def df_metrics_to_num(self, df, query_object):
        """"""
        metrics = [metric for metric in query_object.metrics]
        for col, dtype in df.dtypes.items():
            if dtype.type == np.object_ and col in metrics:
                df[col] = pd.to_numeric(df[col], errors='coerce')
399936
def convert(self, values, nan_rep, encoding, errors):
        

        self.values = Int64Index(np.arange(self.table.nrows))
        return self
625162
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'interval' in _dict:
            args['interval'] = _dict.get('interval')
        if 'event_type' in _dict:
            args['event_type'] = _dict.get('event_type')
        if 'results' in _dict:
            args['results'] = [
                MetricAggregationResult._from_dict(x)
                for x in (_dict.get('results'))
            ]
        return cls(**args)
499999
def encode(df, encoding='utf8', verbosity=1):
    """"""
    if verbosity > 0:
        # pbar_i = 0
        pbar = progressbar.ProgressBar(maxval=df.shape[1])
        pbar.start()
    # encode strings as UTF-8 so they'll work in python2 and python3
    for colnum, col in enumerate(df.columns):
        if isinstance(df[col], pd.Series):
            if verbosity:
                pbar.update(colnum)
            if df[col].dtype in (np.dtype('object'), np.dtype('U'), np.dtype('S')) and any(isinstance(obj, basestring) for obj in df[col]):
                strmask = np.array([isinstance(obj, basestring) for obj in df[col]])
                series = df[col].copy()
                try:
                    series[strmask] = np.char.encode(series[strmask].values.astype('U'))
                except TypeError:
                    print("Unable to convert {} elements starting at position {} in column {}".format(
                          sum(strmask), [i for i, b in enumerate(strmask) if b][:1], col))
                    raise
                except (UnicodeDecodeError, UnicodeEncodeError):
                    try:
                        series[strmask] = np.array([eval(s, {}, {}) for s in series[strmask]])
                    # FIXME: do something different for unicode and decode errors
                    except (SyntaxError, UnicodeDecodeError, UnicodeEncodeError):
                        newseries = []
                        for s in series[strmask]:
                            try:
                                newseries += [s.encode('utf8')]
                            except:
                                print(u'Had trouble encoding {} so used repr to turn it into {}'.format(s, repr(transcode_unicode(s))))
                                # strip all unicode chars are convert to ASCII str
                                newseries += [transcode_unicode(s)]
                        # for dtype('U'): UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 207: ordinal not in r
                        series[strmask] = np.array(newseries).astype('O')
                df[col] = series
                # df[col] = np.array([x.encode('utf8') if isinstance(x, unicode) else x for x in df[col]])

            # WARNING: this takes DAYS for only 100k tweets!
            # series = df[col].copy()
            # for i, value in series.iteritems():   

            #     if isinstance(value, basestring):
            #         series[i] = str(value.encode(encoding))
            # df[col] = series
    if verbosity:
        pbar.finish()
    return df
287852
def convert_numpy_type(cls, dtype):
        

        m = {
            'int64': cls.DATATYPE_INTEGER64,
            'float64': cls.DATATYPE_FLOAT,
            'object': cls.DATATYPE_TEXT  # Hack. Pandas makes strings into object.
        }

        t = m.get(dtype.name, None)

        if not t:
            raise TypeError(
                "Failed to convert numpy type: '{}' ".format(
                    dtype.name))

        return t
354642
def encode_df(df, data_types={}):
    

    numbers = []
    categories = []
    datetimes = []
        
    for column, series in df.iteritems():
        if column in data_types:
            dtypes = data_types[column]
        else:
            dtypes = {}
        
        if 'datetime' in dtypes:
            index = pd.DatetimeIndex(pd.to_datetime(series.values))
            df[column] = index.astype(np.int64).astype(float)
            datetimes.append(column)
            
        elif any([dtype in dtypes for dtype in ['category', 'label']]):
            # If this can be treated as a category, encode it
            df = pd.get_dummies(df,columns=[column])
            categories.append(column)
                    
        elif 'numeric' in dtypes:
            df[column] = clean_numbers(df[column].values)
            df[column] = df[column].astype(float)
            numbers.append(column)
            
        else:
            df.drop(column,1,inplace=True)

    # Scale continuous columns
    scaler = MinMaxScaler()
    df[numbers + datetimes] = scaler.fit_transform(df[numbers + datetimes])
    
    return df, categories, numbers, datetimes
982799
def to_ints(version):
        

        # Example strings: "NSS_3_7_9_RTM", "NSS_3_6_BRANCH_20021026", "NSS_3_6_BETA2",
        #                  "3.18 Basic ECC Beta", "3.16.5 Basic ECC"

        # normalize version strings
        norm_version = version.replace('NSS_', '').replace('.', '_').replace(' ', '_').upper().split('_')

        # Asserting minimumum length of 3 as in [major,minor,tag]
        assert len(norm_version) >= 3

        # Asserting the first two fields are numeric major and minor
        assert norm_version[0].isdigit() and norm_version[1].isdigit()

        # Asserting last field is always a non-numeric tag or a date tag
        # CAVE: fails with obscure date dags like certdata.txt-NSS_3_4_20020403_2
        assert not norm_version[-1].isdigit() or len(norm_version[-1]) > 2

        # fill in missing point and pointpoint versions
        if not (norm_version[2].isdigit() and len(norm_version[2]) < 4):  # <4 to distinguish from numeric date tags
            norm_version.insert(2, "0")
        if not (norm_version[3].isdigit() and len(norm_version[3]) < 4):
            norm_version.insert(3, "0")

        # Strictly ordering by RTM > RC > BETA > *
        # CAVE: Order rule may result in bogus sorting of obscure tags (WITH_CBKI*, TPATCH*, BRANCHPOINT*, ...)
        # Recent versions are tagged non-obscurely and consistently

        tag_value = 0
        for v in norm_version[4:]:
            if v.startswith('BETA'):
                tag_value = 100
                if len(v[4:]) == 1 or len(v[4:]) == 2:
                    try:
                        tag_value += int(v[4:])
                    except ValueError:
                        pass
        for v in norm_version[4:]:
            if v.startswith('RC'):
                tag_value = 200
                if len(v[3:]) == 1 or len(v[3:]) == 2:
                    try:
                        tag_value += int(v[3:])
                    except ValueError:
                        pass
        for v in norm_version[4:]:
            if v == 'RTM':
                tag_value = 300

        # Special case: "x.y.z Basic ECC" is counted as RTM
        # TODO: research the set of potential version string formats reported by libnss.
        if norm_version[-2] == 'BASIC' and norm_version[-1] == 'ECC' and norm_version[-3].isdigit():
            tag_value = 300

        major, minor, point, pointpoint = (int(x) for x in norm_version[:4])

        return [major, minor, point, pointpoint, tag_value]
618248
def _read_sql_with_limit_offset(
    num_splits, sql, con, index_col, kwargs
):  # pragma: no cover
    
    pandas_df = pandas.read_sql(sql, con, index_col=index_col, **kwargs)
    if index_col is None:
        index = len(pandas_df)
    else:
        index = pandas_df.index
    return _split_result_for_readers(1, num_splits, pandas_df) + [index]
1337799
def convert_sklearn_metric_function(scoring):
    """"""
    if callable(scoring):
        module = getattr(scoring, '__module__', None)
        if (
                hasattr(module, 'startswith') and
                module.startswith('sklearn.metrics.') and
                not module.startswith('sklearn.metrics.scorer') and
                not module.startswith('sklearn.metrics.tests.')
        ):
            return make_scorer(scoring)
    return scoring
619706
def convert(values, source_measure_or_unit_abbreviation, target_measure_or_unit_abbreviation):
    

    source_dimension = get_dimension_by_unit_measure_or_abbreviation(source_measure_or_unit_abbreviation)
    target_dimension = get_dimension_by_unit_measure_or_abbreviation(target_measure_or_unit_abbreviation)

    if source_dimension == target_dimension:
        source=JSONObject({})
        target=JSONObject({})
        source.unit_abbreviation, source.factor = _parse_unit(source_measure_or_unit_abbreviation)
        target.unit_abbreviation, target.factor = _parse_unit(target_measure_or_unit_abbreviation)

        source.unit_data = get_unit_by_abbreviation(source.unit_abbreviation)
        target.unit_data = get_unit_by_abbreviation(target.unit_abbreviation)

        source.conv_factor = JSONObject({'lf': source.unit_data.lf, 'cf': source.unit_data.cf})
        target.conv_factor = JSONObject({'lf': target.unit_data.lf, 'cf': target.unit_data.cf})

        if isinstance(values, float):
            # If values is a float => returns a float
            return (source.conv_factor.lf / target.conv_factor.lf * (source.factor * values)
                    + (source.conv_factor.cf - target.conv_factor.cf)
                    / target.conv_factor.lf) / target.factor
        elif isinstance(values, list):
            # If values is a list of floats => returns a list of floats
            return [(source.conv_factor.lf / target.conv_factor.lf * (source.factor * value)
                    + (source.conv_factor.cf - target.conv_factor.cf)
                    / target.conv_factor.lf) / target.factor for value in values]
    else:
        raise HydraError("Unit conversion: dimensions are not consistent.")
1275477
def convert_metric(i)
      case report.metric_type(i)
      when "INTEGER"
        # INTEGER  Integer metric.
        metrics[i].to_i
      when "FLOAT", "PERCENT"
        # FLOAT  Float metric.
        # PERCENT  Percentage metric.
        metrics[i].to_f
      when "CURRENCY"
        # CURRENCY  Currency metric.
        # TODO: Do this better.
        metrics[i].to_f
      when "TIME"
        # Google documentation claims to following:
        # TIME  Time metric in HH:MM:SS format.
        # It also says it's seconds, and that's what I see in real results.
        # So comment out the following:
        # (metrics[i][0..1].to_i +
        #   metrics[i][3..4].to_i * 60 +
        #   metrics[i][6..7].to_i * 24 * 60)
        # Simply make it a float.
        metrics[i].to_f
      else
        # METRIC_TYPE_UNSPECIFIED  Metric type is unspecified.
        metric[i]
      end
    end
249206
def _add_numeric_operations(cls):
        

        def groupby_function(name, alias, npfunc,
                             numeric_only=True, _convert=False,
                             min_count=-1):

            _local_template = "Compute %(f)s of group values"

            @Substitution(name='groupby', f=name)
            @Appender(_common_see_also)
            @Appender(_local_template)
            def f(self, **kwargs):
                if 'numeric_only' not in kwargs:
                    kwargs['numeric_only'] = numeric_only
                if 'min_count' not in kwargs:
                    kwargs['min_count'] = min_count

                self._set_group_selection()
                try:
                    return self._cython_agg_general(
                        alias, alt=npfunc, **kwargs)
                except AssertionError as e:
                    raise SpecificationError(str(e))
                except Exception:
                    result = self.aggregate(
                        lambda x: npfunc(x, axis=self.axis))
                    if _convert:
                        result = result._convert(datetime=True)
                    return result

            set_function_name(f, name, cls)

            return f

        def first_compat(x, axis=0):

            def first(x):
                x = x.to_numpy()

                x = x[notna(x)]
                if len(x) == 0:
                    return np.nan
                return x[0]

            if isinstance(x, DataFrame):
                return x.apply(first, axis=axis)
            else:
                return first(x)

        def last_compat(x, axis=0):

            def last(x):
                x = x.to_numpy()
                x = x[notna(x)]
                if len(x) == 0:
                    return np.nan
                return x[-1]

            if isinstance(x, DataFrame):
                return x.apply(last, axis=axis)
            else:
                return last(x)

        cls.sum = groupby_function('sum', 'add', np.sum, min_count=0)
        cls.prod = groupby_function('prod', 'prod', np.prod, min_count=0)
        cls.min = groupby_function('min', 'min', np.min, numeric_only=False)
        cls.max = groupby_function('max', 'max', np.max, numeric_only=False)
        cls.first = groupby_function('first', 'first', first_compat,
                                     numeric_only=False)
        cls.last = groupby_function('last', 'last', last_compat,
                                    numeric_only=False)
80231
def _convert_bin_to_numeric_type(bins, dtype):
    
    bins_dtype = infer_dtype(bins, skipna=False)
    if is_timedelta64_dtype(dtype):
        if bins_dtype in ['timedelta', 'timedelta64']:
            bins = to_timedelta(bins).view(np.int64)
        else:
            raise ValueError("bins must be of timedelta64 dtype")
    elif is_datetime64_dtype(dtype) or is_datetime64tz_dtype(dtype):
        if bins_dtype in ['datetime', 'datetime64']:
            bins = to_datetime(bins).view(np.int64)
        else:
            raise ValueError("bins must be of datetime64 dtype")

    return bins
135379
def do_metric_statistics(mc, args):
    ''''''
    statistic_types = ['AVG', 'MIN', 'MAX', 'COUNT', 'SUM']
    statlist = args.statistics.split(',')
    for stat in statlist:
        if stat.upper() not in statistic_types:
            errmsg = ('Invalid type, not one of [' +
                      ', '.join(statistic_types) + ']')
            raise osc_exc.CommandError(errmsg)

    fields = {}
    fields['name'] = args.name
    if args.dimensions:
        fields['dimensions'] = utils.format_dimensions_query(args.dimensions)
    _translate_starttime(args)
    fields['start_time'] = args.starttime
    if args.endtime:
        fields['end_time'] = args.endtime
    if args.period:
        fields['period'] = args.period
    fields['statistics'] = args.statistics
    if args.limit:
        fields['limit'] = args.limit
    if args.offset:
        fields['offset'] = args.offset
    if args.merge_metrics:
        fields['merge_metrics'] = args.merge_metrics
    if args.group_by:
        fields['group_by'] = args.group_by
    if args.tenant_id:
        fields['tenant_id'] = args.tenant_id

    try:
        metric = mc.metrics.list_statistics(**fields)
    except (osc_exc.ClientException, k_exc.HttpError) as he:
        raise osc_exc.CommandError('%s\n%s' % (he.message, he.details))
    else:
        if args.json:
            print(utils.json_formatter(metric))
            return
        cols = ['name', 'dimensions']
        # add dynamic column names
        if metric:
            column_names = metric[0]['columns']
            for name in column_names:
                cols.append(name)
        else:
            # when empty set, print_list needs a col
            cols.append('timestamp')

        formatters = {
            'name': lambda x: x['name'],
            'dimensions': lambda x: utils.format_dict(x['dimensions']),
            'timestamp': lambda x:
            format_statistic_timestamp(x['statistics'], x['columns'],
                                       'timestamp'),
            'avg': lambda x:
            format_statistic_value(x['statistics'], x['columns'], 'avg'),
            'min': lambda x:
            format_statistic_value(x['statistics'], x['columns'], 'min'),
            'max': lambda x:
            format_statistic_value(x['statistics'], x['columns'], 'max'),
            'count': lambda x:
            format_statistic_value(x['statistics'], x['columns'], 'count'),
            'sum': lambda x:
            format_statistic_value(x['statistics'], x['columns'], 'sum'),
        }
        if isinstance(metric, list):
            # print the list
            utils.print_list(metric, cols, formatters=formatters)
        else:
            # add the dictionary to a list, so print_list works
            metric_list = list()
            metric_list.append(metric)
            utils.print_list(
                metric_list,
                cols,
                formatters=formatters)
403031
def statsd_metric(name, count, elapsed):
    """"""
    with statsd.pipeline() as pipe:
        pipe.incr(name, count)
        pipe.timing(name, int(round(1000 * elapsed)))
1138580
def numeric_function_clean_dataframe(self, axis):
        
        result = None
        query_compiler = self
        # If no numeric columns and over columns, then return empty Series
        if not axis and len(self.index) == 0:
            result = pandas.Series(dtype=np.int64)

        nonnumeric = [
            col
            for col, dtype in zip(self.columns, self.dtypes)
            if not is_numeric_dtype(dtype)
        ]
        if len(nonnumeric) == len(self.columns):
            # If over rows and no numeric columns, return this
            if axis:
                result = pandas.Series([np.nan for _ in self.index])
            else:
                result = pandas.Series([0 for _ in self.index])
        else:
            query_compiler = self.drop(columns=nonnumeric)
        return result, query_compiler
842840
def format_metric_values(self, metric_values):
    """"""

    # Early in training, metric_values may actually be None.
    loss_str = 'N/A'
    accuracy_str = 'N/A'
    try:
      loss_str = 'loss: %.3f' % metric_values[0]
      accuracy_str = 'accuracy: %.3f' % metric_values[1]
    except (TypeError, IndexError):
      pass

    return '%s, %s' % (loss_str, accuracy_str)