Creates a DataFrame with info about the runs (loaded from the DB)
516686
def create_info_df(self):
        """"""
        logger.debug("running create_info_df")
        # initializing the reader
        reader = self.reader()
        self.info_df = make_df_from_batch(self.name, batch_col=self.batch_col,
                                          reader=reader)
        logger.debug(str(self.info_df.head(5)))
982974
def upload_runsummary(csv_filename, dryrun=False):
    """"""
    print("Checking '{}' for consistency.".format(csv_filename))
    if not os.path.exists(csv_filename):
        log.critical("{} -> file not found.".format(csv_filename))
        return
    try:
        df = pd.read_csv(csv_filename, sep='\t')
    except pd.errors.EmptyDataError as e:
        log.error(e)
        return

    cols = set(df.columns)

    if not REQUIRED_COLUMNS.issubset(cols):
        log.error(
            "Missing columns: {}.".format(
                ', '.join(str(c) for c in REQUIRED_COLUMNS - cols)
            )
        )
        return

    parameters = cols - REQUIRED_COLUMNS
    if len(parameters) < 1:
        log.error("No parameter columns found.")
        return

    if len(df) == 0:
        log.critical("Empty dataset.")
        return

    print(
        "Found data for parameters: {}.".format(
            ', '.join(str(c) for c in parameters)
        )
    )
    print("Converting CSV data into JSON")
    if dryrun:
        log.warn("Dryrun: adding 'TEST_' prefix to parameter names")
        prefix = "TEST_"
    else:
        prefix = ""
    data = convert_runsummary_to_json(df, prefix=prefix)
    print("We have {:.3f} MB to upload.".format(len(data) / 1024**2))

    print("Requesting database session.")
    db = kp.db.DBManager()    # noqa
    if kp.db.we_are_in_lyon():
        session_cookie = "sid=_kmcprod_134.158_lyo7783844001343100343mcprod1223user"    # noqa
    else:
        session_cookie = kp.config.Config().get('DB', 'session_cookie')
        if session_cookie is None:
            raise SystemExit("Could not restore DB session.")
    log.debug("Using the session cookie: {}".format(session_cookie))
    cookie_key, sid = session_cookie.split('=')
    print("Uploading the data to the database.")
    r = requests.post(
        RUNSUMMARY_URL, cookies={cookie_key: sid}, files={'datafile': data}
    )
    if r.status_code == 200:
        log.debug("POST request status code: {}".format(r.status_code))
        print("Database response:")
        db_answer = json.loads(r.text)
        for key, value in db_answer.items():
            print("  -> {}: {}".format(key, value))
        if db_answer['Result'] == 'OK':
            print("Upload successful.")
        else:
            log.critical("Something went wrong.")
    else:
        log.error("POST request status code: {}".format(r.status_code))
        log.critical("Something went wrong...")
        return
1207518
def create_tables(name, runSetResults, rows, rowsDiff, outputPath, outputFilePattern, options):
    '''
    
    '''

    # get common folder of sourcefiles
    common_prefix = os.path.commonprefix([r.filename for r in rows]) # maybe with parts of filename
    common_prefix = common_prefix[: common_prefix.rfind('/') + 1] # only foldername
    for row in rows:
        Row.set_relative_path(row, common_prefix, outputPath)

    # compute nice name of each run set for displaying
    firstBenchmarkName = Util.prettylist(runSetResults[0].attributes['benchmarkname'])
    allBenchmarkNamesEqual = all(Util.prettylist(r.attributes['benchmarkname']) == firstBenchmarkName for r in runSetResults)
    for r in runSetResults:
        if not r.attributes['name']:
            r.attributes['niceName'] = r.attributes['benchmarkname']
        elif allBenchmarkNamesEqual:
            r.attributes['niceName'] = r.attributes['name']
        else:
            r.attributes['niceName'] = [Util.prettylist(r.attributes['benchmarkname']) + '.' + Util.prettylist(r.attributes['name'])]

    template_values = lambda: None # dummy object as dict replacement for simpler syntax
    template_values.head = get_table_head(runSetResults, common_prefix)
    template_values.run_sets = [runSetResult.attributes for runSetResult in runSetResults]
    template_values.columns = [[column for column in runSet.columns] for runSet in runSetResults]
    template_values.columnTitles =\
        [[column.format_title() for column in runSet.columns] for runSet in runSetResults]

    template_values.relevant_id_columns = select_relevant_id_columns(rows)
    template_values.count_id_columns = template_values.relevant_id_columns.count(True)

    template_values.lib_url = options.lib_url
    template_values.base_dir = outputPath
    template_values.href_base = os.path.dirname(options.xmltablefile) if options.xmltablefile else None
    template_values.version = __version__

    futures = []

    def write_table(table_type, title, rows, use_local_summary):
        # calculate statistics if necessary
        if not options.format == ['csv']:
            local_summary = get_summary(runSetResults) if use_local_summary else None
            stats, stats_columns = get_stats(rows, local_summary, options.correct_only)
        else:
            stats = stats_columns = None

        for template_format in (options.format or TEMPLATE_FORMATS):
            if outputFilePattern == '-':
                outfile = None
                logging.info('Writing %s to stdout...', template_format.upper().ljust(4))
            else:
                outfile = os.path.join(outputPath, outputFilePattern.format(name=name, type=table_type, ext=template_format))
                logging.info('Writing %s into %s ...', template_format.upper().ljust(4), outfile)

            this_template_values = dict(title=title, body=rows, foot=stats, foot_columns=stats_columns)
            this_template_values.update(template_values.__dict__)

            futures.append(parallel.submit(
                write_table_in_format,
                template_format, outfile, this_template_values,
                options.show_table and template_format == 'html',
                ))

    # write normal tables
    write_table("table", name, rows,
                use_local_summary=(not options.correct_only and not options.common))

    # write difference tables
    if rowsDiff:
        write_table("diff", name + " differences", rowsDiff, use_local_summary=False)

    return futures
552273
def _get_data_from_dataframe(source, fields='*', first_row=0, count=-1, schema=None):
  
  if schema is None:
    schema = google.datalab.bigquery.Schema.from_data(source)
  fields = get_field_list(fields, schema)
  rows = []
  if count < 0:
    count = len(source.index)
  df_slice = source.reset_index(drop=True)[first_row:first_row + count]
  for index, data_frame_row in df_slice.iterrows():
    row = data_frame_row.to_dict()
    for key in list(row.keys()):
      val = row[key]
      if isinstance(val, pandas.Timestamp):
        row[key] = val.to_pydatetime()

    rows.append({'c': [{'v': row[c]} if c in row else {} for c in fields]})
  cols = _get_cols(fields, schema)
  return {'cols': cols, 'rows': rows}, len(source)
643229
def _build_entity_index(self, run_info, sampling_rate):
        '''  '''

        index = []
        interval = int(round(1000. / sampling_rate))
        _timestamps = []
        for run in run_info:
            reps = int(math.ceil(run.duration * sampling_rate))
            ent_vals = list(run.entities.values())
            df = pd.DataFrame([ent_vals] * reps, columns=list(run.entities.keys()))
            ts = pd.date_range(0, periods=len(df), freq='%sms' % interval)
            _timestamps.append(ts.to_series())
            index.append(df)
        self.timestamps = pd.concat(_timestamps, axis=0, sort=True)
        return pd.concat(index, axis=0, sort=True).reset_index(drop=True)
11371
def _construct_df(pdb_lines):
        """"""
        valids = tuple(pdb_records.keys())
        line_lists = {r: [] for r in valids}
        line_lists['OTHERS'] = []
        for line_num, line in enumerate(pdb_lines):
            if line.strip():
                if line.startswith(valids):
                    record = line[:6].rstrip()
                    line_ele = ['' for _ in range(len(
                        pdb_records[record]) + 1)]
                    for idx, ele in enumerate(pdb_records[record]):
                        line_ele[idx] = (line[ele['line'][0]:ele['line'][1]]
                                         .strip())
                    line_ele[-1] = line_num
                    line_lists[record].append(line_ele)
                else:
                    line_lists['OTHERS'].append([line[:6].rstrip(),
                                                line[6:-1].rstrip(), line_num])

        dfs = {}
        for r in line_lists.items():
            df = pd.DataFrame(r[1], columns=[c['id'] for c in
                                             pdb_records[r[0]]] + ['line_idx'])
            for c in pdb_records[r[0]]:
                try:
                    df[c['id']] = df[c['id']].astype(c['type'])
                except ValueError:
                    # expect ValueError if float/int columns are empty strings
                    df[c['id']] = pd.Series(np.nan, index=df.index)

            dfs[r[0]] = df
        return dfs
1031009
def make_table(self):
        """"""
        num_records = int(np.sum([1 for frame in self.timeseries]))
        dtype = [("frame",float),("time",float),("ligand atom id",int),
                ("ligand atom name","|U4"),("distance",float),
                ("resid",int),("resname","|U4"),("segid","|U8") ]
        out = np.empty((num_records,),dtype=dtype)
        cursor=0
        for contact in self.timeseries:
            out[cursor] = (contact.frame, contact.time,contact.ligandatomid,contact.ligandatomname,contact.distance,
                           contact.resid,contact.resname,contact.segid)
            cursor+=1
        return out.view(np.recarray)
1034692
def to_dataframe(self):
        """"""
        def row_from_variant(variant):
            return OrderedDict([
                ("chr", variant.contig),
                ("start", variant.original_start),
                ("ref", variant.original_ref),
                ("alt", variant.original_alt),
                ("gene_name", ";".join(variant.gene_names)),
                ("gene_id", ";".join(variant.gene_ids))
            ])
        rows = [row_from_variant(v) for v in self]
        if len(rows) == 0:
            # TODO: return a DataFrame with the appropriate columns
            return pd.DataFrame()
        return pd.DataFrame.from_records(rows, columns=rows[0].keys())
19792
def load_info_df(self, file_name=None):
        """"""

        if file_name is None:
            file_name = self.info_file

        with open(file_name, 'r') as infile:
            top_level_dict = json.load(infile)

        new_info_df_dict = top_level_dict['info_df']
        new_info_df = pd.DataFrame(new_info_df_dict)
        self.info_df = new_info_df

        self._prm_packer(top_level_dict['metadata'])
        self.info_file = file_name
        logger.debug("loaded info_df")
        logger.debug(" info_file: %s" % self.info_file)
100777
def run(self):
        

        history = pd.DataFrame()

        # get history from csv dir
        if self.backtest and self.backtest_csv:
            kind = "TICK" if self.resolution[-1] in ("S", "K", "V") else "BAR"
            dfs = []
            for symbol in self.symbols:
                file = "%s/%s.%s.csv" % (self.backtest_csv, symbol, kind)
                if not os.path.exists(file):
                    self.log_algo.error(
                        "Can't load data for %s (%s doesn't exist)",
                        symbol, file)
                    sys.exit(0)
                try:
                    df = pd.read_csv(file)
                    if "expiry" not in df.columns:
                        df.loc[:, "expiry"] = nan

                    if not validate_csv_columns(df, kind, raise_errors=False):
                        self.log_algo.error(
                            "%s isn't a QTPyLib-compatible format", file)
                        sys.exit(0)

                    if df['symbol'].values[-1] != symbol:
                        self.log_algo.error(
                            "%s Doesn't content data for %s", file, symbol)
                        sys.exit(0)

                    dfs.append(df)

                except Exception as e:
                    self.log_algo.error(
                        "Error reading data for %s (%s)", symbol, file)
                    sys.exit(0)

            history = prepare_history(
                data=pd.concat(dfs, sort=True),
                resolution=self.resolution,
                tz=self.timezone,
                continuous=self.continuous
            )
            history = history[history.index >= self.backtest_start]

        elif not self.blotter_args["dbskip"] and (
                self.backtest or self.preload):

            start = self.backtest_start if self.backtest else tools.backdate(
                self.preload)
            end = self.backtest_end if self.backtest else None

            history = self.blotter.history(
                symbols=self.symbols,
                start=start,
                end=end,
                resolution=self.resolution,
                tz=self.timezone,
                continuous=self.continuous
            )

            # history needs backfilling?
            # self.blotter.backfilled = True
            if not self.blotter.backfilled:
                # "loan" Blotter our ibConn
                self.blotter.ibConn = self.ibConn

                # call the back fill
                self.blotter.backfill(data=history,
                                      resolution=self.resolution,
                                      start=start, end=end)

                # re-get history from db
                history = self.blotter.history(
                    symbols=self.symbols,
                    start=start,
                    end=end,
                    resolution=self.resolution,
                    tz=self.timezone,
                    continuous=self.continuous
                )

                # take our ibConn back :)
                self.blotter.ibConn = None

        # optimize pandas
        if not history.empty:
            history['symbol'] = history['symbol'].astype('category')
            history['symbol_group'] = history['symbol_group'].astype('category')
            history['asset_class'] = history['asset_class'].astype('category')

        if self.backtest:
            # initiate strategy
            self.on_start()

            # drip history
            drip_handler = self._tick_handler if self.resolution[-1] in (
                "S", "K", "V") else self._bar_handler
            self.blotter.drip(history, drip_handler)

        else:
            # place history self.bars
            self.bars = history

            # add instruments to blotter in case they do not exist
            self.blotter.register(self.instruments)

            # initiate strategy
            self.on_start()

            # listen for RT data
            self.blotter.stream(
                symbols=self.symbols,
                tz=self.timezone,
                quote_handler=self._quote_handler,
                tick_handler=self._tick_handler,
                bar_handler=self._bar_handler,
                book_handler=self._book_handler
            )
942348
def tcounts(self):
        
        df = pd.DataFrame([[t.name(), t.size()] for t in self.tables()], columns=["name", "size"])
        df.index = df.name
        return df
577683
def as_dataframe(self, fillna=True, subjects=None):
        
        entries = []
        selected_subjects = self.subjects
        if subjects is not None:
            selected_subjects = subjects
            
        for s in selected_subjects:
            vmap = {}
            for c in self.inferred_types(s):
                vmap[c] = 1
            entries.append(vmap)
        logging.debug("Creating DataFrame")
        df = pd.DataFrame(entries, index=selected_subjects)
        if fillna:
            logging.debug("Performing fillna...")
            df = df.fillna(0)
        return df
222957
def dataframe(self):
        
        if self._points_allowed is None and self._points_scored is None:
            return None
        fields_to_include = {
            'boxscore_index': self.boxscore_index,
            'date': self.date,
            'datetime': self.datetime,
            'game': self.game,
            'location': self.location,
            'losses': self.losses,
            'opponent_abbr': self.opponent_abbr,
            'opponent_name': self.opponent_name,
            'points_allowed': self.points_allowed,
            'points_scored': self.points_scored,
            'result': self.result,
            'streak': self.streak,
            'time': self.time,
            'wins': self.wins
        }
        return pd.DataFrame([fields_to_include], index=[self._boxscore])
376822
def table(self, table):
        
        return DataFrame.from_spark_rdd(self.sql_ctx.table(table),
                                        self.sql_ctx)
421486
def to_instants_dataframe(self, sql_ctx):
        
        ssql_ctx = sql_ctx._ssql_ctx
        jdf = self._jtsrdd.toInstantsDataFrame(ssql_ctx, -1)
        return DataFrame(jdf, sql_ctx)
508016
def dataset(self):
        '''
        
        '''
        target = self.db.target_table
        db_examples = self.db.rows(target, [self.db.target_att, self.db.pkeys[target]])

        examples = []
        for cls, pk in sorted(db_examples, key=lambda ex: ex[0]):
            facts = self._facts(pk, self.db.pkeys[target], target, visited=set())
            examples.append('%s %s' % (cls, ', '.join(facts)))

        return '\n'.join(examples)
59480
def dataframe(self):
        
        if self._points_for is None and self._points_against is None:
            return None
        fields_to_include = {
            'boxscore_index': self.boxscore_index,
            'date': self.date,
            'datetime': self.datetime,
            'day_of_week': self.day_of_week,
            'game': self.game,
            'location': self.location,
            'losses': self.losses,
            'opponent_abbr': self.opponent_abbr,
            'opponent_conference': self.opponent_conference,
            'opponent_name': self.opponent_name,
            'opponent_rank': self.opponent_rank,
            'points_against': self.points_against,
            'points_for': self.points_for,
            'rank': self.rank,
            'result': self.result,
            'streak': self.streak,
            'time': self.time,
            'wins': self.wins
        }
        return pd.DataFrame([fields_to_include], index=[self._boxscore])