Runs a query against Druid and returns a dataframe.
342703
def run_query(  # noqa / druid
            self,
            groupby, metrics,
            granularity,
            from_dttm, to_dttm,
            filter=None,  # noqa
            is_timeseries=True,
            timeseries_limit=None,
            timeseries_limit_metric=None,
            row_limit=None,
            inner_from_dttm=None, inner_to_dttm=None,
            orderby=None,
            extras=None,  # noqa
            columns=None, phase=2, client=None,
            order_desc=True,
            prequeries=None,
            is_prequery=False,
        ):
        
        # TODO refactor into using a TBD Query object
        client = client or self.cluster.get_pydruid_client()
        row_limit = row_limit or conf.get('ROW_LIMIT')

        if not is_timeseries:
            granularity = 'all'

        if granularity == 'all':
            phase = 1
        inner_from_dttm = inner_from_dttm or from_dttm
        inner_to_dttm = inner_to_dttm or to_dttm

        timezone = from_dttm.replace(tzinfo=DRUID_TZ).tzname() if from_dttm else None

        query_str = ''
        metrics_dict = {m.metric_name: m for m in self.metrics}
        columns_dict = {c.column_name: c for c in self.columns}

        if (
            self.cluster and
            LooseVersion(self.cluster.get_druid_version()) < LooseVersion('0.11.0')
        ):
            for metric in metrics:
                self.sanitize_metric_object(metric)
            self.sanitize_metric_object(timeseries_limit_metric)

        aggregations, post_aggs = DruidDatasource.metrics_and_post_aggs(
            metrics,
            metrics_dict)

        self.check_restricted_metrics(aggregations)

        # the dimensions list with dimensionSpecs expanded
        dimensions = self.get_dimensions(groupby, columns_dict)
        extras = extras or {}
        qry = dict(
            datasource=self.datasource_name,
            dimensions=dimensions,
            aggregations=aggregations,
            granularity=DruidDatasource.granularity(
                granularity,
                timezone=timezone,
                origin=extras.get('druid_time_origin'),
            ),
            post_aggregations=post_aggs,
            intervals=self.intervals_from_dttms(from_dttm, to_dttm),
        )

        filters = DruidDatasource.get_filters(filter, self.num_cols, columns_dict)
        if filters:
            qry['filter'] = filters

        having_filters = self.get_having_filters(extras.get('having_druid'))
        if having_filters:
            qry['having'] = having_filters

        order_direction = 'descending' if order_desc else 'ascending'

        if columns:
            columns.append('__time')
            del qry['post_aggregations']
            del qry['aggregations']
            qry['dimensions'] = columns
            qry['metrics'] = []
            qry['granularity'] = 'all'
            qry['limit'] = row_limit
            client.scan(**qry)
        elif len(groupby) == 0 and not having_filters:
            logging.info('Running timeseries query for no groupby values')
            del qry['dimensions']
            client.timeseries(**qry)
        elif (
                not having_filters and
                len(groupby) == 1 and
                order_desc
        ):
            dim = list(qry.get('dimensions'))[0]
            logging.info('Running two-phase topn query for dimension [{}]'.format(dim))
            pre_qry = deepcopy(qry)
            if timeseries_limit_metric:
                order_by = utils.get_metric_name(timeseries_limit_metric)
                aggs_dict, post_aggs_dict = DruidDatasource.metrics_and_post_aggs(
                    [timeseries_limit_metric],
                    metrics_dict)
                if phase == 1:
                    pre_qry['aggregations'].update(aggs_dict)
                    pre_qry['post_aggregations'].update(post_aggs_dict)
                else:
                    pre_qry['aggregations'] = aggs_dict
                    pre_qry['post_aggregations'] = post_aggs_dict
            else:
                agg_keys = qry['aggregations'].keys()
                order_by = list(agg_keys)[0] if agg_keys else None

            # Limit on the number of timeseries, doing a two-phases query
            pre_qry['granularity'] = 'all'
            pre_qry['threshold'] = min(row_limit,
                                       timeseries_limit or row_limit)
            pre_qry['metric'] = order_by
            pre_qry['dimension'] = self._dimensions_to_values(qry.get('dimensions'))[0]
            del pre_qry['dimensions']

            client.topn(**pre_qry)
            logging.info('Phase 1 Complete')
            if phase == 2:
                query_str += '// Two phase query\n// Phase 1\n'
            query_str += json.dumps(
                client.query_builder.last_query.query_dict, indent=2)
            query_str += '\n'
            if phase == 1:
                return query_str
            query_str += (
                "// Phase 2 (built based on phase one's results)\n")
            df = client.export_pandas()
            qry['filter'] = self._add_filter_from_pre_query_data(
                df,
                [pre_qry['dimension']],
                filters)
            qry['threshold'] = timeseries_limit or 1000
            if row_limit and granularity == 'all':
                qry['threshold'] = row_limit
            qry['dimension'] = dim
            del qry['dimensions']
            qry['metric'] = list(qry['aggregations'].keys())[0]
            client.topn(**qry)
            logging.info('Phase 2 Complete')
        elif len(groupby) > 0 or having_filters:
            # If grouping on multiple fields or using a having filter
            # we have to force a groupby query
            logging.info('Running groupby query for dimensions [{}]'.format(dimensions))
            if timeseries_limit and is_timeseries:
                logging.info('Running two-phase query for timeseries')

                pre_qry = deepcopy(qry)
                pre_qry_dims = self._dimensions_to_values(qry['dimensions'])

                # Can't use set on an array with dicts
                # Use set with non-dict items only
                non_dict_dims = list(
                    set([x for x in pre_qry_dims if not isinstance(x, dict)]),
                )
                dict_dims = [x for x in pre_qry_dims if isinstance(x, dict)]
                pre_qry['dimensions'] = non_dict_dims + dict_dims

                order_by = None
                if metrics:
                    order_by = utils.get_metric_name(metrics[0])
                else:
                    order_by = pre_qry_dims[0]

                if timeseries_limit_metric:
                    order_by = utils.get_metric_name(timeseries_limit_metric)
                    aggs_dict, post_aggs_dict = DruidDatasource.metrics_and_post_aggs(
                        [timeseries_limit_metric],
                        metrics_dict)
                    if phase == 1:
                        pre_qry['aggregations'].update(aggs_dict)
                        pre_qry['post_aggregations'].update(post_aggs_dict)
                    else:
                        pre_qry['aggregations'] = aggs_dict
                        pre_qry['post_aggregations'] = post_aggs_dict

                # Limit on the number of timeseries, doing a two-phases query
                pre_qry['granularity'] = 'all'
                pre_qry['limit_spec'] = {
                    'type': 'default',
                    'limit': min(timeseries_limit, row_limit),
                    'intervals': self.intervals_from_dttms(
                        inner_from_dttm, inner_to_dttm),
                    'columns': [{
                        'dimension': order_by,
                        'direction': order_direction,
                    }],
                }
                client.groupby(**pre_qry)
                logging.info('Phase 1 Complete')
                query_str += '// Two phase query\n// Phase 1\n'
                query_str += json.dumps(
                    client.query_builder.last_query.query_dict, indent=2)
                query_str += '\n'
                if phase == 1:
                    return query_str
                query_str += (
                    "// Phase 2 (built based on phase one's results)\n")
                df = client.export_pandas()
                qry['filter'] = self._add_filter_from_pre_query_data(
                    df,
                    pre_qry['dimensions'],
                    filters,
                )
                qry['limit_spec'] = None
            if row_limit:
                dimension_values = self._dimensions_to_values(dimensions)
                qry['limit_spec'] = {
                    'type': 'default',
                    'limit': row_limit,
                    'columns': [{
                        'dimension': (
                            utils.get_metric_name(
                                metrics[0],
                            ) if metrics else dimension_values[0]
                        ),
                        'direction': order_direction,
                    }],
                }
            client.groupby(**qry)
            logging.info('Query Complete')
        query_str += json.dumps(
            client.query_builder.last_query.query_dict, indent=2)
        return query_str
1049960
def table_as_df(self, table_name):
        """"""
        self.table_must_exist(table_name)
        query = "SELECT * FROM `%s`" % table_name.lower()
        return pandas.read_sql(query, self.own_conn)
1010571
def make_dataframe(obj, columns=None, exclude=None, limit=1e8):
    """"""
    try:
        obj = obj.objects.all()[:limit]
    except:
        pass
    if isinstance(obj, (pd.Series, list, tuple)):
        return make_dataframe(pd.DataFrame(obj), columns, exclude, limit)
    # if the obj is a named tuple, DataFrame, dict of columns, django QuerySet, sql alchemy query result
    # retrieve the "include"d field/column names from its keys/fields/attributes
    if columns is None:
        columns = get_column_labels(obj)
    if exclude is not None and columns is not None and columns and exclude:
        columns = [i for i in columns if i not in exclude]
    try:
        return pd.DataFrame(list(obj.values(*columns)[:limit]))
    except:
        pass
    try:
        return pd.DataFrame(obj)[fuzzy_get(obj, columns)]
    except:
        pass
    return pd.DataFrame(obj)
149089
def _get_dataframe(self):
        
        if self.source_conn is None:  # Use local file
            return ge.read_csv(self.dataset_name, **self.dataset_params)

        if isinstance(self.source_conn, S3Hook):
            hook = ExpectationS3CsvHook(aws_conn_id=self.source_conn_id)

            return hook.get_ge_df(self.dataset_name, self.source_bucket_name, **self.dataset_params)

        if isinstance(self.source_conn, DbApiHook):
            hook = ExpectationMySQLHook(mysql_conn_id=self.source_conn_id)

            return hook.get_ge_df(self.dataset_name, **self.dataset_params)
241743
def query(self, expr, **kwargs):
        
        columns = self.columns

        def query_builder(df, **kwargs):
            # This is required because of an Arrow limitation
            # TODO revisit for Arrow error
            df = df.copy()
            df.index = pandas.RangeIndex(len(df))
            df.columns = columns
            df.query(expr, inplace=True, **kwargs)
            df.columns = pandas.RangeIndex(len(df.columns))
            return df

        func = self._prepare_method(query_builder, **kwargs)
        new_data = self._map_across_full_axis(1, func)
        # Query removes rows, so we need to update the index
        new_index = self.compute_index(0, new_data, True)

        return self.__constructor__(new_data, new_index, self.columns, self.dtypes)
38772
def toDF(self, *cols):
        
        jdf = self._jdf.toDF(self._jseq(cols))
        return DataFrame(jdf, self.sql_ctx)
1169698
def getquery(query):
	''
	try:
		conn = connection.cursor()
		conn.execute(query)
		data = conn.fetchall()
		conn.close()
	except: data = list()
	return data
69909
def find(self, *args, **kwargs):
    " "
    return Query(*args, db=self, schema=self.schema)
981050
def run(self):
        """"""
        query = self.query

        # count before filtering
        self.cardinality = query.add_columns(self.columns[0].sqla_expr).count()

        self._set_column_filter_expressions()
        self._set_global_filter_expression()
        self._set_sort_expressions()
        self._set_yadcf_data(query)

        # apply filters
        query = query.filter(
            *[e for e in self.filter_expressions if e is not None])

        self.cardinality_filtered = query.add_columns(
            self.columns[0].sqla_expr).count()

        # apply sorts
        query = query.order_by(
            *[e for e in self.sort_expressions if e is not None])

        # add paging options
        length = int(self.params.get('length'))
        if length >= 0:
            query = query.limit(length)
        elif length == -1:
            pass
        else:
            raise (ValueError(
                'Length should be a positive integer or -1 to disable'))
        query = query.offset(int(self.params.get('start')))

        # add columns to query
        query = query.add_columns(*[c.sqla_expr for c in self.columns])

        # fetch the result of the queries
        column_names = [
            col.mData if col.mData else str(i)
            for i, col in enumerate(self.columns)
        ]
        self.results = [{k: v
                         for k, v in zip(column_names, row)}
                        for row in query.all()]
1102904
def get_query_result(self, query_object):
        """"""

        # Here, we assume that all the queries will use the same datasource, which is
        # is a valid assumption for current setting. In a long term, we may or maynot
        # support multiple queries from different data source.

        timestamp_format = None
        if self.datasource.type == 'table':
            dttm_col = self.datasource.get_col(query_object.granularity)
            if dttm_col:
                timestamp_format = dttm_col.python_date_format

        # The datasource here can be different backend but the interface is common
        result = self.datasource.query(query_object.to_dict())

        df = result.df
        # Transform the timestamp we received from database to pandas supported
        # datetime format. If no python_date_format is specified, the pattern will
        # be considered as the default ISO date format
        # If the datetime format is unix, the parse will use the corresponding
        # parsing logic
        if df is not None and not df.empty:
            if DTTM_ALIAS in df.columns:
                if timestamp_format in ('epoch_s', 'epoch_ms'):
                    # Column has already been formatted as a timestamp.
                    df[DTTM_ALIAS] = df[DTTM_ALIAS].apply(pd.Timestamp)
                else:
                    df[DTTM_ALIAS] = pd.to_datetime(
                        df[DTTM_ALIAS], utc=False, format=timestamp_format)
                if self.datasource.offset:
                    df[DTTM_ALIAS] += timedelta(hours=self.datasource.offset)
                df[DTTM_ALIAS] += query_object.time_shift

            if self.enforce_numerical_metrics:
                self.df_metrics_to_num(df, query_object)

            df.replace([np.inf, -np.inf], np.nan)
        return {
            'query': result.query,
            'status': result.status,
            'error_message': result.error_message,
            'df': df,
        }
361150
def timeseries(self, dataframe=False):
        

        self.query.get_cardinality("author_uuid").by_period()
        return super().timeseries(dataframe)
1284086
def apply(self, data_source):
        
        dataframe = self.__get_dataframe(data_source, use_target=False)
        dataframe = self.__cleaner.apply(dataframe)
        dataframe = self.__transformer.apply(dataframe)
        return dataframe
808060
def to_dataframe(self, columns=None):
    """"""
    if not self.is_value_set:
      raise ValueError('Value must be set before converting to a DataFrame.')
    if not pandas:
      raise RuntimeError('Install pandas to convert to pandas.DataFrame')
    return pandas.DataFrame.from_records(self.value, columns=columns)
1307019
def get_query(query_id, session, retry_count=5):
    """"""
    query = None
    attempt = 0
    while not query and attempt < retry_count:
        try:
            query = session.query(Query).filter_by(id=query_id).one()
        except Exception:
            attempt += 1
            logging.error(
                'Query with id `{}` could not be retrieved'.format(query_id))
            stats_logger.incr('error_attempting_orm_query_' + str(attempt))
            logging.error('Sleeping for a sec before retrying...')
            sleep(1)
    if not query:
        stats_logger.incr('error_failed_at_getting_orm_query')
        raise SqlLabException('Failed at getting query')
    return query
552273
def _get_data_from_dataframe(source, fields='*', first_row=0, count=-1, schema=None):
  
  if schema is None:
    schema = google.datalab.bigquery.Schema.from_data(source)
  fields = get_field_list(fields, schema)
  rows = []
  if count < 0:
    count = len(source.index)
  df_slice = source.reset_index(drop=True)[first_row:first_row + count]
  for index, data_frame_row in df_slice.iterrows():
    row = data_frame_row.to_dict()
    for key in list(row.keys()):
      val = row[key]
      if isinstance(val, pandas.Timestamp):
        row[key] = val.to_pydatetime()

    rows.append({'c': [{'v': row[c]} if c in row else {} for c in fields]})
  cols = _get_cols(fields, schema)
  return {'cols': cols, 'rows': rows}, len(source)
788381
def execute(query, data=None):
    
    connection = _State.connection()
    _State.new_transaction()

    if data is None:
        data = []

    result = connection.execute(query, data)

    _State.table = None
    _State.metadata = None
    try:
        del _State.table_pending
    except AttributeError:
        pass

    if not result.returns_rows:
        return {u'data': [], u'keys': []}

    return {u'data': result.fetchall(), u'keys': list(result.keys())}
464959
def load_django(self, query: "django query"):
        
        try:
            self.df = self._load_django(query)
        except Exception as e:
            self.err(e, "Can not load data from query")