Iterates through the sparql table and condenses it into a Pandas DataFrame
355586
def get_sparql_dataframe( self ):
        '''  '''
        self.result = self.g.query(self.query)
        cols = set() # set(['qname'])
        indx = set()
        data = {}
        curr_subj = None # place marker for first subj to be processed
        bindings = []

        for i, binding in enumerate(self.result.bindings):
            subj_binding = binding[rdflib.term.Variable('subj')]
            pred_binding = binding[rdflib.term.Variable('pred')]
            obj_binding  = binding[rdflib.term.Variable('obj')]

            subj = subj_binding
            pred = pred_binding # self.qname(pred_binding) if self.predicate_as_qname else pred_binding
            obj  = obj_binding

            # stops at BNodes; could be exanded here
            if isinstance(subj, BNode):
                continue
            elif isinstance(pred, BNode):
                continue
            elif isinstance(obj, BNode) and obj:
                continue
            # else:
            #     subj = str(subj)
            #     pred = str(pred)
            #     obj  = str(obj)

            cols.add(pred)
            indx.add(subj)
            bindings.append(binding)

        bindings = sorted(bindings, key=lambda k: k[rdflib.term.Variable('subj')])
        df = pd.DataFrame(columns=cols, index=indx)

        for i, binding in enumerate(bindings):

            subj_binding = binding[rdflib.term.Variable('subj')]
            pred_binding = binding[rdflib.term.Variable('pred')]
            obj_binding  = binding[rdflib.term.Variable('obj')]

            subj = subj_binding
            pred = pred_binding # self.qname(pred_binding) if self.qname_predicates else pred_binding
            obj  = obj_binding

            # stops at BNodes; could be exanded here
            if isinstance(subj, BNode):
                continue
            elif isinstance(pred, BNode):
                continue
            elif isinstance(obj, BNode) and obj:
                continue
            # elif self.value_type:
            #     subj = str(subj)
            #     pred = str(pred)
            #     obj  = str(obj)

            if curr_subj == None:
                curr_subj = subj
                if not data.get(subj): # Prepare defaultdict home if it doesn't exist
                    data[subj] = defaultdict(list)
                data[subj][pred].append(obj)
            elif curr_subj != subj:
                curr_subj = subj
                for data_subj, data_pred_objs in data.items():
                    for data_pred, data_objs in data_pred_objs.items():
                        if len(data_objs) == 1: # clean lists of just 1 value
                            data_pred_objs[data_pred] = data_objs[0]
                    df.loc[data_subj] = pd.Series(data_pred_objs)
                data = {}
                if not data.get(subj): # Prepare defaultdict home if it doesn't exist
                    data[subj] = defaultdict(list)
                data[subj][pred].append(obj)
            else:
                if not data.get(subj): # Prepare defaultdict home if it doesn't exist
                    data[subj] = defaultdict(list)
                data[subj][pred].append(obj)

        for data_subj, data_pred_objs in data.items():
            for data_pred, data_objs in data_pred_objs.items():
                if len(data_objs) == 1: # clean lists of just 1 value
                    data_pred_objs[data_pred] = data_objs[0]
            df.loc[data_subj] = pd.Series(data_pred_objs)

        df = df.where((pd.notnull(df)), None) # default Null is fricken Float NaN

        return df
1115992
def dataframe_from_result_table(table):
    import pandas as pd
    from ._models import KustoResultTable
    from dateutil.tz import UTC

    
    if not table:
        raise ValueError()

    if not isinstance(table, KustoResultTable):
        raise TypeError("Expected KustoResultTable got {}".format(type(table).__name__))

    columns = [col.column_name for col in table.columns]
    frame = pd.DataFrame(table._rows, columns=columns)

    # fix types
    for col in table.columns:
        if col.column_type == "bool":
            frame[col.column_name] = frame[col.column_name].astype(bool)

    return frame
90824
def parse_results(fields, data):
    
    master = []

    for record in data['records']:  # for each 'record' in response
        row = [None] * len(fields)  # create null list the length of number of columns
        for obj, value in record.iteritems():  # for each obj in record
            if not isinstance(value, (dict, list, tuple)):  # if not data structure
                if obj in fields:
                    row[fields.index(obj)] = ensure_utf(value)

            elif isinstance(value, dict) and obj != 'attributes':  # traverse down into object
                path = obj
                _traverse_results(value, fields, row, path)

        master.append(row)
    return master
616290
def get_rows(self):
        
        possible_dataframes = ['F', 'FY', 'M', 'S',
                               'D_cba', 'D_pba',  'D_imp', 'D_exp',
                               'D_cba_reg', 'D_pba_reg',
                               'D_imp_reg', 'D_exp_reg',
                               'D_cba_cap', 'D_pba_cap',
                               'D_imp_cap', 'D_exp_cap', ]
        for df in possible_dataframes:
            if (df in self.__dict__) and (getattr(self, df) is not None):
                return getattr(self, df).index.get_values()
        else:
            logging.warn("No attributes available to get row names")
            return None
965980
def to_dataframe(self, **kwargs):
        """"""
        return pandas.io.parsers.read_csv(self.path, sep=self.d, **kwargs)
55052
def RDFValueToBigQuerySchema(self, value):
    """"""
    fields_array = []
    for type_info in value.__class__.type_infos:
      # Nested structures are indicated by setting type "RECORD"
      if type_info.__class__.__name__ == "ProtoEmbedded":
        fields_array.append({
            "name": type_info.name,
            "type": "RECORD",
            "description": type_info.description,
            "fields": self.RDFValueToBigQuerySchema(value.Get(type_info.name))
        })
      else:
        # If we don't have a specific map use string.
        bq_type = self.RDF_BIGQUERY_TYPE_MAP.get(type_info.proto_type_name,
                                                 None) or "STRING"

        # For protos with RDF types we need to do some more checking to properly
        # covert types.
        if hasattr(type_info, "original_proto_type_name"):
          if type_info.original_proto_type_name in [
              "RDFDatetime", "RDFDatetimeSeconds"
          ]:
            bq_type = "TIMESTAMP"
          elif type_info.proto_type_name == "uint64":
            # This is to catch fields like st_mode which are stored as ints but
            # exported as more useful strings. Things which are just plain ints
            # won't have an RDF type specified and so will be exported as
            # INTEGER
            bq_type = "STRING"

        fields_array.append({
            "name": type_info.name,
            "type": bq_type,
            "description": type_info.description
        })
    return fields_array
770077
def condense(self):
        
        for pseudo_key, rows in self._rows.items():
            tmp1 = []
            intervals = sorted(self._derive_distinct_intervals(rows))
            for interval in intervals:
                tmp2 = dict(zip(self._pseudo_key, pseudo_key))
                tmp2[self._key_start_date] = interval[0]
                tmp2[self._key_end_date] = interval[1]
                tmp1.append(tmp2)

            self._rows[pseudo_key] = tmp1
1093238
def db_from_dataframe(
        db_filename,
        table_name,
        df,
        primary_key=None,
        subdir=None,
        overwrite=False,
        indices=(),
        version=1):
    
    return db_from_dataframes(
        db_filename=db_filename,
        dataframes={table_name: df},
        primary_keys={table_name: primary_key},
        indices={table_name: indices},
        subdir=subdir,
        overwrite=overwrite,
        version=version)
811475
def _make_repr_table_from_sframe(X):
    

    assert isinstance(X, _SFrame)

    column_names = X.column_names()

    out_data = [ [None]*len(column_names) for i in range(X.num_rows())]

    column_sizes = [len(s) for s in column_names]

    for i, c in enumerate(column_names):
        for j, e in enumerate(X[c]):
            out_data[j][i] = str(e)
            column_sizes[i] = max(column_sizes[i], len(e))

    # now, go through and pad everything.
    out_data = ([ [cn.ljust(k, ' ') for cn, k in zip(column_names, column_sizes)],
                  ["-"*k for k in column_sizes] ]
                + [ [e.ljust(k, ' ') for e, k in zip(row, column_sizes)] for row in out_data] )

    return ['  '.join(row) for row in out_data]
1049960
def table_as_df(self, table_name):
        """"""
        self.table_must_exist(table_name)
        query = "SELECT * FROM `%s`" % table_name.lower()
        return pandas.read_sql(query, self.own_conn)
1321768
def lipd_to_df(metadata, csvs):
    
    dfs = {}
    logger_dataframes.info("enter lipd_to_df")

    # Flatten the dictionary, but ignore the chron data items
    dict_in_dotted = {}
    logger_dataframes.info("enter dot_notation")
    _dotnotation_for_nested_dictionary(metadata, '', dict_in_dotted)
    dict_in_dotted = collections.OrderedDict(sorted(dict_in_dotted.items()))

    # Create one data frame for metadata items
    dfs["metadata"] = pd.DataFrame(list(dict_in_dotted.items()), columns=["Key", "Value"])

    # Create data frames for paleo data and chron data items. This does not use LiPD data, it uses the csv data
    dfs.update(_get_dfs(csvs))

    return dfs
508016
def dataset(self):
        '''
        
        '''
        target = self.db.target_table
        db_examples = self.db.rows(target, [self.db.target_att, self.db.pkeys[target]])

        examples = []
        for cls, pk in sorted(db_examples, key=lambda ex: ex[0]):
            facts = self._facts(pk, self.db.pkeys[target], target, visited=set())
            examples.append('%s %s' % (cls, ', '.join(facts)))

        return '\n'.join(examples)
1033166
def to_sql(self, app=None, last_known_value_date: Optional[date] = None):
        

        from delphi.apps.rest_api import create_app, db

        self.assign_uuids_to_nodes_and_edges()
        icm_metadata = ICMMetadata(
            id=self.id,
            created=self.dateCreated,
            estimatedNumberOfPrimitives=len(self.nodes) + len(self.edges),
            createdByUser_id=1,
            lastAccessedByUser_id=1,
            lastUpdatedByUser_id=1,
        )
        if last_known_value_date is None:
            today = date.today().isoformat()
        else:
            today = last_known_value_date.isoformat()
        default_latent_var_value = 1.0
        causal_primitives = []
        nodeset = {n.split("/")[-1] for n in self.nodes}
        simplified_labels = len(nodeset) == len(self)
        for n in self.nodes(data=True):
            n[1]["rv"] = LatentVar(n[0])
            n[1]["update_function"] = self.default_update_function
            rv = n[1]["rv"]
            rv.dataset = [default_latent_var_value for _ in range(self.res)]

            causal_variable = CausalVariable(
                id=n[1]["id"],
                model_id=self.id,
                units="",
                namespaces={},
                auxiliaryProperties=[],
                label=n[0].split("/")[-1].replace("_", " ").capitalize()
                if simplified_labels
                else n[0],
                description=n[0],
                lastUpdated=today,
                confidence=1.0,
                lastKnownValue={
                    "active": "ACTIVE",
                    "trend": None,
                    "time": today,
                    "value": {
                        "baseType": "FloatValue",
                        "value": n[1]["rv"].dataset[0],
                    },
                },
                range={
                    "baseType": "FloatRange",
                    "range": {"min": 0, "max": 5, "step": 0.1},
                },
            )
            causal_primitives.append(causal_variable)

        max_mean_betas = max(
            [abs(np.median(e[2]["βs"])) for e in self.edges(data=True)]
        )
        for e in self.edges(data=True):
            causal_relationship = CausalRelationship(
                id=e[2]["id"],
                namespaces={},
                source={
                    "id": self.nodes[e[0]]["id"],
                    "baseType": "CausalVariable",
                },
                target={
                    "id": self.nodes[e[1]]["id"],
                    "baseType": "CausalVariable",
                },
                model_id=self.id,
                auxiliaryProperties=[],
                lastUpdated=today,
                types=["causal"],
                description="",
                confidence=np.mean(
                    [s.belief for s in e[2]["InfluenceStatements"]]
                ),
                label="",
                strength=abs(np.median(e[2]["βs"]) / max_mean_betas),
                reinforcement=(
                    True
                    if np.mean(
                        [
                            stmt.subj_delta["polarity"]
                            * stmt.obj_delta["polarity"]
                            for stmt in e[2]["InfluenceStatements"]
                        ]
                    )
                    > 0
                    else False
                ),
            )
            causal_primitives.append(causal_relationship)
        evidences = []
        for edge in self.edges(data=True):
            for stmt in edge[2]["InfluenceStatements"]:
                for ev in stmt.evidence:
                    evidence = Evidence(
                        id=str(uuid4()),
                        causalrelationship_id=edge[2]["id"],
                        # TODO - make source and target appear in CauseEx HMI
                        description=(ev.text),
                    )
                    evidences.append(evidence)

        if app is None:
            app = create_app()

        with app.app_context():
            db.drop_all()
            db.create_all()
            db.session.add(icm_metadata)
            db.session.add(DelphiModel(id=self.id, model=self))
            for causal_primitive in causal_primitives:
                db.session.add(causal_primitive)
            for evidence in evidences:
                db.session.add(evidence)
            db.session.commit()
66173
def parse(resp) -> DataFrameType:
    """"""
    statements = []
    for statement in resp['results']:
        series = {}
        for s in statement.get('series', []):
            series[_get_name(s)] = _drop_zero_index(_serializer(s))
        statements.append(series)

    if len(statements) == 1:
        series: dict = statements[0]
        if len(series) == 1:
            return list(series.values())[0]  # DataFrame
        else:
            return series  # dict
    return statements
586320
def fetch_result_sets(cls, db, datasource_type):
        
        result_set_df = db.get_df(
            """SELECT table_schema, table_name FROM INFORMATION_SCHEMA.{}S
               ORDER BY concat(table_schema, '.', table_name)""".format(
                datasource_type.upper(),
            ),
            None)
        result_sets = []
        for unused, row in result_set_df.iterrows():
            result_sets.append('{}.{}'.format(
                row['table_schema'], row['table_name']))
        return result_sets
1371475
def dcounts(self):
        
        print("WARNING: Distinct value count for all tables can take a long time...", file=sys.stderr)
        sys.stderr.flush()

        data = []
        for t in self.tables():
            for c in t.columns():
                data.append([t.name(), c.name(), c.dcount(), t.size(), c.dcount() / float(t.size())])
        df = pd.DataFrame(data, columns=["table", "column", "distinct", "size", "fraction"])
        return df
713410
def _build_dataframes(self):
        """"""

        for component in self.all_components:

            attrs = self.components[component]["attrs"]

            static_dtypes = attrs.loc[attrs.static, "dtype"].drop(["name"])

            df = pd.DataFrame({k: pd.Series(dtype=d) for k, d in static_dtypes.iteritems()},
                              columns=static_dtypes.index)

            df.index.name = "name"

            setattr(self,self.components[component]["list_name"],df)

            pnl = Dict({k : pd.DataFrame(index=self.snapshots,
                                         columns=[],
                                         #it's currently hard to imagine non-float series, but this could be generalised
                                         dtype=np.dtype(float))
                        for k in attrs.index[attrs.varying]})

            setattr(self,self.components[component]["list_name"]+"_t",pnl)