Memory efficient function for loading a table from a FITS
    file.
644
def create_table_from_fits(fitsfile, hduname, colnames=None):
    """"""

    if colnames is None:
        return Table.read(fitsfile, hduname)

    cols = []
    with fits.open(fitsfile, memmap=True) as h:
        for k in colnames:
            data = h[hduname].data.field(k)
            cols += [Column(name=k, data=data)]
    return Table(cols)
408436
def _fromfile(self):
        
        fd = self.parent._fd
        byte_order = self.parent.byte_order
        offset_size = self.parent.offset_size

        fmt = {4: 'I', 8: 'Q'}[offset_size]
        offset = struct.unpack(byte_order + fmt, fd.read(offset_size))[0]
        if not offset:
            raise StopIteration()

        # read standard tags
        tags = self.tags
        fd.seek(offset)
        fmt, size = {4: ('H', 2), 8: ('Q', 8)}[offset_size]
        try:
            numtags = struct.unpack(byte_order + fmt, fd.read(size))[0]
        except Exception:
            warnings.warn("corrupted page list")
            raise StopIteration()

        for _ in range(numtags):
            tag = TIFFtag(self.parent)
            tags[tag.name] = tag

        # read LSM info subrecords
        if self.is_lsm:
            pos = fd.tell()
            for name, reader in CZ_LSM_INFO_READERS.items():
                try:
                    offset = self.cz_lsm_info["offset_"+name]
                except KeyError:
                    continue
                if not offset:
                    continue
                fd.seek(offset)
                try:
                    setattr(self, "cz_lsm_"+name, reader(fd, byte_order))
                except ValueError:
                    pass
            fd.seek(pos)
314623
def prep_fastq_inputs(in_files, data):
    
    if len(in_files) == 1 and _is_bam_input(in_files):
        out = _bgzip_from_bam(in_files[0], data["dirs"], data)
    elif len(in_files) == 1 and _is_cram_input(in_files):
        out = _bgzip_from_cram(in_files[0], data["dirs"], data)
    elif len(in_files) in [1, 2] and _ready_gzip_fastq(in_files, data):
        out = _symlink_in_files(in_files, data)
    else:
        if len(in_files) > 2:
            fpairs = fastq.combine_pairs(in_files)
            pair_types = set([len(xs) for xs in fpairs])
            assert len(pair_types) == 1
            fpairs.sort(key=lambda x: os.path.basename(x[0]))
            organized = [[xs[0] for xs in fpairs]]
            if len(fpairs[0]) > 1:
                organized.append([xs[1] for xs in fpairs])
            in_files = organized
        parallel = {"type": "local", "num_jobs": len(in_files),
                    "cores_per_job": max(1, data["config"]["algorithm"]["num_cores"] // len(in_files))}
        inputs = [{"in_file": x, "read_num": i, "dirs": data["dirs"], "config": data["config"],
                   "is_cwl": "cwl_keys" in data,
                   "rgnames": data["rgnames"]}
                  for i, x in enumerate(in_files) if x]
        out = run_multicore(_bgzip_from_fastq_parallel, [[d] for d in inputs], data["config"], parallel)
    return out
1064830
def _fileToMatrix(file_name):
    """"""
    # TODO: np.loadtxt() might be an alternative
    #     try:
    if 1 < 3:
        lres = []
        for line in open(file_name, 'r').readlines():
            if len(line) > 0 and line[0] not in ('%', '#'):
                lres.append(list(map(float, line.split())))
        res = lres
    while res != [] and res[0] == []:  # remove further leading empty lines
        del res[0]
    return res
    #     except:
    print('could not read file ' + file_name)
251606
def _load_general(data, targets, major_axis):
    """"""
    for d_src, d_targets, axis in zip(data, targets, major_axis): # pylint: disable=too-many-nested-blocks
        if isinstance(d_targets, nd.NDArray):
            d_src.copyto(d_targets)
        elif isinstance(d_src, (list, tuple)):
            for src, dst in zip(d_src, d_targets):
                src.copyto(dst)
        else:
            for slice_idx, d_dst in d_targets:
                if axis >= 0:
                    # copy slice
                    shape = d_src.shape
                    do_crop = (slice_idx.start != 0 or shape[axis] != slice_idx.stop)
                    # pylint: disable=no-member,protected-access
                    if do_crop:
                        if axis == 0:
                            d_src[slice_idx.start:slice_idx.stop].copyto(d_dst)
                        else:
                            if d_src.context == d_dst.context:
                                nd.slice_axis(d_src, axis=axis, begin=slice_idx.start,
                                              end=slice_idx.stop, out=d_dst)
                            else:
                                # on different device, crop and then do cross device copy
                                d_dst_copy = nd.slice_axis(d_src, axis=axis, begin=slice_idx.start,
                                                           end=slice_idx.stop)
                                d_dst_copy.copyto(d_dst)
                    else:
                        d_src.copyto(d_dst)
                    # pylint: enable=no-member,protected-access
                else:
                    d_src.copyto(d_dst)
588505
def table_repr(columns, rows, data, padding=2):
    """"""
    padding = ' ' * padding
    column_lengths = [len(column) for column in columns]
    for row in rows:
        for i, column in enumerate(columns):
            item = str(data[row][column])
            column_lengths[i] = max(len(item), column_lengths[i])
    max_row_length = max(len(row) for row in rows) if len(rows) else 0
    table_row = ' ' * max_row_length
    for i, column in enumerate(columns):
        table_row += padding + column.rjust(column_lengths[i])
    table_rows = [table_row]
    for row in rows:
        table_row = row.rjust(max_row_length)
        for i, column in enumerate(columns):
            item = str(data[row][column])
            table_row += padding + item.rjust(column_lengths[i])
        table_rows.append(table_row)
    return '\n'.join(table_rows)
808222
def _load_data(path, labels_number=1):
  """"""
  with tf.io.gfile.GFile(path, "rb") as f:
    data = f.read()
  offset = 0
  max_offset = len(data) - 1
  while offset < max_offset:
    labels = np.frombuffer(data, dtype=np.uint8, count=labels_number,
                           offset=offset).reshape((labels_number,))
    # 1 byte per label, 1024 * 3 = 3072 bytes for the image.
    offset += labels_number
    img = (np.frombuffer(data, dtype=np.uint8, count=3072, offset=offset)
           .reshape((3, _CIFAR_IMAGE_SIZE, _CIFAR_IMAGE_SIZE))
           .transpose((1, 2, 0))
          )
    offset += 3072
    yield labels, img
968934
def load_data(verbose=False):
    ""

    df = pd.read_csv(STOPS_PATH, usecols=['stop_id', 'stop_name'])
    if verbose:
        print('- Loaded %d entries from "%s".' % (len(df), STOPS_PATH))
    return df
1122858
def load_table(self, table, rows, colnames=None, limit=None):
        

        conn = self.engine.raw_connection()
        cur = conn.cursor()

        colnames = colnames or next(rows)

        cmd = 'INSERT INTO {table} ({colnames}) VALUES ({placeholders})'.format(
            table=self.tables[table],
            colnames=', '.join(colnames),
            placeholders=', '.join([self.placeholder] * len(colnames)))

        cur.executemany(cmd, itertools.islice(rows, limit))
        conn.commit()
668860
def load_contents(self):
        
        with open(METADATA_FILE) as f:
            lines = f.readlines()

        lines = map(lambda x: x.strip(), lines)

        exclude_strings = ['<begin_table>', '<end_table>']

        list_of_databases_and_columns = filter(
            lambda x: not x[0] in exclude_strings, [
                list(value) for key, value in itertools.groupby(
                    lines,
                    lambda x: x in exclude_strings
                )
            ]
        )

        for iterator in list_of_databases_and_columns:
            self.create_table_raw(
                tablename=iterator[0],
                columns=iterator[1:][:],
            )

        for i in self.tables:
            i.load_contents()
1294184
def process_fasta(fasta, **kwargs):
    """"""
    logging.info("Nanoget: Starting to collect statistics from a fasta file.")
    inputfasta = handle_compressed_input(fasta, file_type="fasta")
    return ut.reduce_memory_usage(pd.DataFrame(
        data=[len(rec) for rec in SeqIO.parse(inputfasta, "fasta")],
        columns=["lengths"]
    ).dropna())
1273816
def _load(self, dataset_spec):
        """"""

        for idx, ds in enumerate(dataset_spec):
            self.append(ds, idx)
1089895
def write_table_file(self, table_file=None):
        """"""
        if self._table is None:
            raise RuntimeError("No table to write")
        if table_file is not None:
            self._table_file = table_file
        if self._table_file is None:
            raise RuntimeError("No output file specified for table")
        write_tables_to_fits(self._table_file, [self._table], clobber=True,
                             namelist=['FILE_ARCHIVE'])
859944
def load(self, filename):
        """"""
        f = io.open(filename,'r',encoding='utf-8')
        for line in f:
            data = line.strip().split("\t")
            type, count = data[:2]
            self.count(type,count)
        f.close()
715601
def load_from_file(filename):
    '''
    
    '''
    t = None
    with open(filename, 'rU') as infile:
        tdata = infile.read()
        t = Topology.unserialize(tdata)
    return t
1363649
def create_from_fits(cls, fitsfile, norm_type='flux'):
        
        tsmap = WcsNDMap.read(fitsfile)

        tab_e = Table.read(fitsfile, 'EBOUNDS')
        tab_s = Table.read(fitsfile, 'SCANDATA')
        tab_f = Table.read(fitsfile, 'FITDATA')

        tab_e = convert_sed_cols(tab_e)
        tab_s = convert_sed_cols(tab_s)
        tab_f = convert_sed_cols(tab_f)

        emin = np.array(tab_e['e_min'])
        emax = np.array(tab_e['e_max'])
        try:
            if str(tab_e['e_min'].unit) == 'keV':
                emin /= 1000.
        except:
            pass
        try:
            if str(tab_e['e_max'].unit) == 'keV':
                emax /= 1000.
        except:
            pass

        nebins = len(tab_e)
        npred = tab_e['ref_npred']

        ndim = len(tsmap.data.shape)

        if ndim == 2:
            cube_shape = (tsmap.data.shape[0],
                          tsmap.data.shape[1], nebins)
        elif ndim == 1:
            cube_shape = (tsmap.data.shape[0], nebins)
        else:
            raise RuntimeError("Counts map has dimension %i" % (ndim))

        refSpec = ReferenceSpec.create_from_table(tab_e)
        nll_vals = -np.array(tab_s["dloglike_scan"])
        norm_vals = np.array(tab_s["norm_scan"])

        axis = MapAxis.from_edges(np.concatenate((emin, emax[-1:])),
                                  interp='log')
        geom_3d = tsmap.geom.to_cube([axis])
        tscube = WcsNDMap(geom_3d,
                          np.rollaxis(tab_s["ts"].reshape(cube_shape), 2, 0))

        ncube = WcsNDMap(geom_3d,
                         np.rollaxis(tab_s["norm"].reshape(cube_shape), 2, 0))
        nmap = WcsNDMap(tsmap.geom,
                        tab_f['fit_norm'].reshape(tsmap.data.shape))

        ref_colname = 'ref_%s' % norm_type
        norm_vals *= tab_e[ref_colname][np.newaxis, :, np.newaxis]

        return cls(tsmap, nmap, tscube, ncube,
                   norm_vals, nll_vals, refSpec,
                   norm_type)
457697
def load_data_table(file,
                    index_col,
                    show_progress=False):
    
    with ZipFile(file) as zip_file:
        file_names = zip_file.namelist()
        assert len(file_names) == 1, "Expected a single file from Quandl."
        wiki_prices = file_names.pop()
        with zip_file.open(wiki_prices) as table_file:
            if show_progress:
                log.info('Parsing raw data.')
            data_table = pd.read_csv(
                table_file,
                parse_dates=['date'],
                index_col=index_col,
                usecols=[
                    'ticker',
                    'date',
                    'open',
                    'high',
                    'low',
                    'close',
                    'volume',
                    'ex-dividend',
                    'split_ratio',
                ],
            )

    data_table.rename(
        columns={
            'ticker': 'symbol',
            'ex-dividend': 'ex_dividend',
        },
        inplace=True,
        copy=False,
    )
    return data_table