Iteratively read data from a GTFS table. Returns namedtuples.
762086
def iterread(self, table):
    """"""
    self.log('Reading: %s'%table)
    # Entity class
    cls = self.FACTORIES[table]
    f = self._open(table)
    # csv reader
    if unicodecsv:
      data = unicodecsv.reader(f, encoding='utf-8-sig')
    else:
      data = csv.reader(f)
    header = data.next()
    headerlen = len(header)
    ent = collections.namedtuple(
      'EntityNamedTuple',
      map(str, header)
    )
    for row in data:
      if len(row) == 0:
        continue
      # Get rid of extra spaces.
      row = [i.strip() for i in row]
      # pad to length if necessary... :(
      if len(row) < headerlen:
        row += ['']*(headerlen-len(row))
      yield cls.from_row(ent._make(row), self)
    f.close()
320563
def read_table(self):
        """"""
        self.bitcount = self.bits = 0
        tlen = unpack('>I', self.input.read(4))[0]
        table_data = AMQPReader(self.input.read(tlen))
        result = {}
        while table_data.input.tell() < tlen:
            name = table_data.read_shortstr()
            val = table_data.read_item()
            result[name] = val
        return result
596422
def get_gtf_db(gtf, in_memory=False):
    
    db_file = gtf + '.db'
    if gtf.endswith('.gz'):
        db_file = gtf[:-3] + '.db'
    if file_exists(db_file):
        return gffutils.FeatureDB(db_file)
    db_file = ':memory:' if in_memory else db_file
    if in_memory or not file_exists(db_file):
        debug('GTF database does not exist, creating...')
        infer_extent = guess_infer_extent(gtf)
        db = gffutils.create_db(gtf, dbfn=db_file,
                                infer_gene_extent=infer_extent)
        return db
    else:
        return gffutils.FeatureDB(db_file)
1263417
def to_named_tuple(keys, values):
    

    values = [dbobject.__dict__[key] for key in dbobject.keys()]

    tuple_object = namedtuple('DBObject', dbobject.keys())

    tuple_instance = tuple_object._make(values)

    return tuple_instance
534062
def read_table(self, table_type):
        """"""

        if table_type == 'hash':
            entry_class = MPQHashTableEntry
        elif table_type == 'block':
            entry_class = MPQBlockTableEntry
        else:
            raise ValueError("Invalid table type.")

        table_offset = self.header['%s_table_offset' % table_type]
        table_entries = self.header['%s_table_entries' % table_type]
        key = self._hash('(%s table)' % table_type, 'TABLE')

        self.file.seek(table_offset + self.header['offset'])
        data = self.file.read(table_entries * 16)
        data = self._decrypt(data, key)

        def unpack_entry(position):
            entry_data = data[position*16:position*16+16]
            return entry_class._make(
                struct.unpack(entry_class.struct_format, entry_data))

        return [unpack_entry(i) for i in range(table_entries)]
790887
def get_goea_nts_all(self, fldnames=None, **kws):
        
        # kws: prt_if indent itemid2name(study_items)
        data_nts = [] # A list of namedtuples containing GOEA results
        if not self.goea_results:
            return data_nts
        keep_if = kws.get('keep_if', None)
        rpt_fmt = kws.get('rpt_fmt', False)
        indent = kws.get('indent', False)
        # I. FIELD (column) NAMES
        not_fldnames = kws.get('not_fldnames', None)
        if fldnames is None:
            fldnames = self._get_fieldnames(self.goea_results[0])
        # Ia. Explicitly exclude specific fields from named tuple
        if not_fldnames is not None:
            fldnames = [f for f in fldnames if f not in not_fldnames]
        nttyp = cx.namedtuple("NtGoeaResults", " ".join(fldnames))
        goid_idx = fldnames.index("GO") if 'GO' in fldnames else None
        # II. Loop through GOEA results stored in a GOEnrichmentRecord object
        for goerec in self.goea_results:
            vals = self._get_field_values(goerec, fldnames, rpt_fmt, kws.get('itemid2name', None))
            if indent:
                vals[goid_idx] = "".join([goerec.get_indent_dots(), vals[goid_idx]])
            ntobj = nttyp._make(vals)
            if keep_if is None or keep_if(goerec):
                data_nts.append(ntobj)
        return data_nts
144001
def load_transcript_fpkm_dict_from_gtf(
        gtf_path,
        transcript_id_column_name="reference_id",
        fpkm_column_name="FPKM",
        feature_column_name="feature"):
    
    df = gtfparse.read_gtf(
        gtf_path,
        column_converters={fpkm_column_name: float})
    transcript_ids = _get_gtf_column(transcript_id_column_name, gtf_path, df)
    fpkm_values = _get_gtf_column(fpkm_column_name, gtf_path, df)
    features = _get_gtf_column(feature_column_name, gtf_path, df)
    logging.info("Loaded %d rows from %s" % (len(transcript_ids), gtf_path))
    logging.info("Found %s transcript entries" % sum(
        feature == "transcript" for feature in features))
    result = {
        transcript_id: float(fpkm)
        for (transcript_id, fpkm, feature)
        in zip(transcript_ids, fpkm_values, features)
        if (
            (transcript_id is not None) and
            (len(transcript_id) > 0) and
            (feature == "transcript")
        )
    }
    logging.info("Keeping %d transcript rows with reference IDs" % (
        len(result),))
    return result
310449
def _read_table_file(self, table_file):
        """"""
        self._table_file = table_file
        if os.path.exists(self._table_file):
            self._table = Table.read(self._table_file, hdu='JOB_ARCHIVE')
            self._table_ids = Table.read(self._table_file, hdu='FILE_IDS')
        else:
            self._table, self._table_ids = JobDetails.make_tables({})
        self._table_id_array = self._table_ids['file_id'].data
        self._fill_cache()
1374102
def zip_namedtuple(nt_list):
    
    if not nt_list:
        return dict()
    if not isinstance(nt_list, list):
        nt_list = [nt_list]
    for nt in nt_list:
        assert type(nt) == type(nt_list[0])
    ret = {k : [v] for k, v in nt_list[0]._asdict().items()}
    for nt in nt_list[1:]:
        for k, v in nt._asdict().items():
            ret[k].append(v)
    return ret
1103216
def parse_hist_files(histf):
        """"""
        nameddata = dict()
        data = dict()
        try:
            for l in histf['f'].splitlines():
                s = l.split()
                if s:
                    if len(s) != 2:
                        raise RuntimeError("invalid format: " + str(len(s)) + " column(s) found in row. must be exactly 2.")
                    data[int(s[0])] = int(s[1])
        except Exception as err:
            log.warning("Error parsing %s. %s", histf['fn'], err)
            log.debug(traceback.format_exc())
        else:
            if data:
                nameddata[histf['s_name']] = data
            else:
                log.debug("%s is empty.", histf['fn'])
        finally:
            return nameddata
731428
def read_data( self, step, filename = None ):
        """"""
        filename = get_default( filename, self.filename )

        out = {}

        fd = open( self.filename, 'r' )
        while 1:
            line = skip_read_line(fd, no_eof=True).split()
            if line[0] == 'POINT_DATA':
                break

        n_nod = int(line[1])

        while 1:
            line = skip_read_line(fd)
            if not line:
                break

            line = line.split()

            if line[0] == 'SCALARS':
                name, dtype, nc = line[1:]
                assert_(int(nc) == 1)
                fd.readline() # skip lookup table line
                
                data = nm.zeros((n_nod,), dtype=nm.float64)
                ii = 0
                while ii < n_nod:
                    data[ii] = float(fd.readline())
                    ii += 1

                out[name] = Struct( name = name,
                                    mode = 'vertex',
                                    data = data,
                                    dofs = None )

            elif line[0] == 'VECTORS':
                name, dtype = line[1:]
                data = []
                ii = 0
                while ii < n_nod:
                    data.append([float(val) for val in fd.readline().split()])
                    ii += 1

                out[name] = Struct( name = name,
                                    mode = 'vertex',
                                    data = nm.array(data, dtype=nm.float64),
                                    dofs = None )

            elif line[0] == 'CELL_DATA':
                break

            line = fd.readline()

        fd.close()

        return out
850919
def get_namedtuple_factory(cls, field_mappings, name="Record"):
    
    ntup = cls.get_namedtuple(field_mappings, name)
    return lambda data: ntup(**data)
39858
def packed_parallel_tsv_dataset(filenames=gin.REQUIRED,
                                dataset_split=gin.REQUIRED,
                                batch_size=gin.REQUIRED,
                                sequence_length=gin.REQUIRED,
                                vocabulary=gin.REQUIRED,
                                append_eos=True,
                                shuffle_buffer_size=10000,
                                eos_id=1):
  """"""

  dataset = tf.data.TextLineDataset(filenames)
  if dataset_split == "train":
    dataset = dataset.repeat()
    dataset = dataset.shuffle(shuffle_buffer_size)

  def _parse_fn(record):  # pylint: disable=missing-docstring
    tokens = tf.decode_csv(
        record,
        record_defaults=[""] * 2,
        field_delim="\t",
        use_quote_delim=False)
    return {"inputs": tokens[0], "targets": tokens[1]}

  def _encode_fn(features):  # pylint: disable=missing-docstring
    inputs_vocabulary = vocabulary[0] if isinstance(vocabulary,
                                                    tuple) else vocabulary
    targets_vocabulary = vocabulary[1] if isinstance(vocabulary,
                                                     tuple) else vocabulary
    inputs_enc = inputs_vocabulary.encode_tf(features["inputs"])
    targets_enc = targets_vocabulary.encode_tf(features["targets"])
    if append_eos:
      inputs_enc = tf.concat([tf.to_int64(inputs_enc), [eos_id]], 0)
      targets_enc = tf.concat([tf.to_int64(targets_enc), [eos_id]], 0)
    return {"inputs": inputs_enc, "targets": targets_enc}

  dataset = dataset.map(_parse_fn)
  dataset = dataset.map(_encode_fn)
  return pack_and_batch(dataset, batch_size, sequence_length)
1308899
def load_data(subseqs):
        """"""
        print('            . load_data')
        lines = Lines()
        lines.add(1, 'cpdef inline void load_data(self, int idx) %s:' % _nogil)
        lines.add(2, 'cdef int jdx0, jdx1, jdx2, jdx3, jdx4, jdx5')
        for seq in subseqs:
            lines.add(2, 'if self._%s_diskflag:' % seq.name)
            if seq.NDIM == 0:
                lines.add(3, 'fread(&self.%s, 8, 1, self._%s_file)'
                             % (2*(seq.name,)))
            else:
                lines.add(3, 'fread(&self.%s[0], 8, self._%s_length, '
                             'self._%s_file)' % (3*(seq.name,)))
            lines.add(2, 'elif self._%s_ramflag:' % seq.name)
            if seq.NDIM == 0:
                lines.add(3, 'self.%s = self._%s_array[idx]' % (2*(seq.name,)))
            else:
                indexing = ''
                for idx in range(seq.NDIM):
                    lines.add(3+idx, 'for jdx%d in range(self._%s_length_%d):'
                                     % (idx, seq.name, idx))
                    indexing += 'jdx%d,' % idx
                indexing = indexing[:-1]
                lines.add(3+seq.NDIM, 'self.%s[%s] = self._%s_array[idx,%s]'
                                      % (2*(seq.name, indexing)))
        return lines
1371770
def dicts_from_table(table, keys=None):
    """"""
    lod = []
    for row in table:
        if not keys:
            keys = row
            if not all((k not in (None, '') or bool(k.strip())) for k in keys):
                keys = None
            continue
        lod += [OrderedDict((k, v) for k, v in zip(keys, row))]
    return lod
1192129
def to_named_tuple(obj, visited_children=None, back_relationships=None, levels=None, ignore=[], extras={}):
    

    if visited_children is None:
        visited_children = []

    if back_relationships is None:
        back_relationships = []

    serialized_data = {c.key: getattr(obj, c.key) for c in obj.__table__.columns}


    #Any other non-column data to include in the keyed tuple
    for k, v in extras.items():
        serialized_data[k] = v

    relationships = class_mapper(obj.__class__).relationships

    #Set the attributes to 'None' first, so the attributes are there, even if they don't
    #get filled in:
    for name, relation in relationships.items():
        if relation.uselist:
            serialized_data[name] = tuple([])
        else:
            serialized_data[name] = None


    visitable_relationships = [(name, rel) for name, rel in relationships.items() if name not in back_relationships]

    if levels is not None and levels > 0:
        for name, relation in visitable_relationships:

            levels = levels - 1

            if name in ignore:
                continue

            if relation.backref:
                back_relationships.append(relation.backref)

            relationship_children = getattr(obj, name)

            if relationship_children is not None:
                if relation.uselist:
                    children = []
                    for child in [c for c in relationship_children if c not in visited_children]:
                        visited_children.append(child)
                        children.append(to_named_tuple(child, visited_children, back_relationships, ignore=ignore, levels=levels))
                    serialized_data[name] = tuple(children)
                else:
                    serialized_data[name] = to_named_tuple(relationship_children, visited_children, back_relationships, ignore=ignore, levels=levels)

    vals = []
    cols = []
    for k, v in serialized_data.items():
        vals.append(k)
        cols.append(v)

    result = KeyedTuple(cols, vals)

    return result
112422
def _get_gtf_column(column_name, gtf_path, df):
    
    if column_name in df.columns:
        return list(df[column_name])

    else:
        raise ValueError(
            "Missing '%s' in columns of %s, available: %s" % (
                column_name,
                gtf_path,
                list(df.columns)))