Read CSV delimited input from Slurm.
352675
def _read_output(self, command):
        
        cmd = []
        cmd.extend(self._prefix)
        cmd.extend([self._path, "-iP"])
        cmd.extend(command)
        command = cmd

        logger.debug("Cmd %s" % command)
        null = open('/dev/null', 'w')
        process = subprocess.Popen(
            command, stdout=subprocess.PIPE, stderr=null)
        null.close()

        results = []
        reader = csv.reader(_input_csv(process.stdout), delimiter=str("|"))

        try:
            headers = next(reader)
            logger.debug("<-- headers %s" % headers)
        except StopIteration:
            logger.debug("Cmd %s headers not found" % command)
            headers = []

        for row in reader:
            _output_csv(row)

            logger.debug("<-- row %s" % row)
            this_row = {}

            i = 0
            for i in range(0, len(headers)):
                key = headers[i]
                value = row[i]
                this_row[key] = value

            results.append(this_row)

        process.stdout.close()
        retcode = process.wait()
        if retcode != 0:
            logger.error("<-- Cmd %s returned %d (error)" % (command, retcode))
            raise subprocess.CalledProcessError(retcode, command)

        if len(headers) == 0:
            logger.error("Cmd %s didn't return any headers." % command)
            raise RuntimeError("Cmd %s didn't return any headers." % command)

        logger.debug("<-- Returned: %d (good)" % retcode)
        return results
223662
def read_file(filename):
    
    dtype = np.dtype([('same_cell', np.int32),
                      ('N1', np.int),
                      ('N2', np.int),
                      ('time', np.float)
                      ])
    if pd is not None:
        timings = pd.read_csv(filename, header=None,
                              engine="c",
                              dtype={'same_cell': np.int32,
                                     'N1': np.int,
                                     'N2': np.int,
                                     'time': np.float},
                              index_col=None,
                              names=['same_cell', 'N1', 'N2', 'time'],
                              delim_whitespace=True)
    else:
        timings = np.loadtxt(filename, dtype=dtype)
    return timings
791657
def csv_to_list(csv_file, delimiter=','):
    
    with open_csv(csv_file) as csv_con:
        if len(delimiter) > 1:
            dialect = csv.Sniffer().sniff(csv_con.read(1024),
                                          delimiters=delimiter)
            csv_con.seek(0)
            reader = csv.reader(csv_con, dialect)
        else:
            reader = csv.reader(csv_con, delimiter=delimiter)
        return list(reader)
1074848
def read_csv(self, csv_location, csv_configs=None):
        # type: (str, Dict[str, int]) -> List[List[str]]
        
        csv_configs = self._generate_configs_from_default(csv_configs)
        with open(csv_location, 'r') as csv_file:
            csv_reader = csv.reader(csv_file)
            self.csv_data = list(csv_reader)
        self.csv_data = self.csv_data[csv_configs['HEADER_COLUMNS_TO_SKIP']:]
        return self.csv_data
471990
def read_csv(filepath):
    
    symbols = []
    with open(filepath, 'rb') as csvfile:
        spamreader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
        for row in spamreader:
            symbols.append(row)
    return symbols
711990
def read_unicode_csv_fileobj(fileobj, delimiter=',', quotechar='"',
                             quoting=csv.QUOTE_MINIMAL, lineterminator='\n',
                             encoding='utf-8', skiprows=0):
    """"""
    # Python 3 version
    if sys.version_info[0] >= 3:
        # Next, get the csv reader, with unicode delimiter and quotechar
        csv_reader = csv.reader(fileobj, delimiter=delimiter,
                                quotechar=quotechar, quoting=quoting,
                                lineterminator=lineterminator)
        # Now, return the (already decoded) unicode csv_reader generator
        # Skip rows if necessary
        for skip_ix in range(skiprows):
            next(csv_reader)
        for row in csv_reader:
            yield row
    # Python 2 version
    else:
        # Next, get the csv reader, passing delimiter and quotechar as
        # bytestrings rather than unicode
        csv_reader = csv.reader(fileobj, delimiter=delimiter.encode(encoding),
                             quotechar=quotechar.encode(encoding),
                             quoting=quoting, lineterminator=lineterminator)
        # Iterate over the file and decode each string into unicode
        # Skip rows if necessary
        for skip_ix in range(skiprows):
            next(csv_reader)
        for row in csv_reader:
            yield [cell.decode(encoding) for cell in row]
915597
def map_to_csv(fields, delimiter=","):
    

    def _list_to_csv(l):
        """
        Util function to overcome the use of files by in-memory io buffer

        :param l:
        :return:
        """
        io_file = io.StringIO()
        writer = csv.writer(io_file, quoting=csv.QUOTE_NONNUMERIC, lineterminator='', delimiter=delimiter)
        writer.writerow(l)
        return io_file.getvalue()

    def _app(current_tuple, e=None):
        if e is not None:
            return None, e
        csv_list = []
        for f in fields:
            if f in current_tuple:
                csv_list.append(current_tuple[f])
            else:
                e.update({"output": "expected field {} not found".format(f)})
                return None, e
        return _list_to_csv(csv_list), e
    if fields is None or len(fields) == 0:
        return fixed_input(None, "no fields provided, cannot proceed without order")
    return _app
460587
def csv_to_pickle(path=ROOT / "raw", clean=False):
    """"""

    for file in os.listdir(path):
        base, ext = os.path.splitext(file)
        if ext != ".csv":
            continue

        LOG(f"converting {file} to pickle")
        df = pd.read_csv(path / file, low_memory=True)
        WRITE_DF(df, path / (base + "." + FILE_EXTENSION), **WRITE_DF_OPTS)
        if clean:
            os.remove(path / file)
            LOG(f"removed {file}")
590789
def _read(self, ddfile):
        
        import csv

        d = {}
        with open(ddfile) as f:
            reader = csv.DictReader(f)
            for row in reader:
                key = row['name']
                if key.startswith('#'):
                    key = key[1:]
                    deprecated = True
                else:
                    deprecated = False
                value = d.setdefault(key, {})
                value[self._get_value(row)] = deprecated
        return d
990827
def parse_sacct(sacct_stream):
    """"""
    rows = (line.split() for line in sacct_stream)
    # filter rows that begin with a SLURM job id
    relevant_rows = (row for row in rows if row[0].isdigit())
    jobs = [convert_job(row) for row in relevant_rows]
    return jobs
1208741
def read_csv(fpath):
    
    import csv
    import utool as ut
    #csvfile = open(fpath, 'rb')
    with open(fpath, 'rb') as csvfile:
        row_iter = csv.reader(csvfile, delimiter=str(','), quotechar=str('|'))
        row_list = [ut.lmap(ut.ensure_unicode, row) for row in row_iter]
    return row_list
845662
def csv_to_dicts(file, header=None):
    """"""
    with open(file) as csvfile:
        return [row for row in csv.DictReader(csvfile, fieldnames=header)]
181111
def csv_to_map(fields, delimiter=','):
    

    def _csv_to_list(csv_input):
        """
        Util function to overcome the use of files by in-memory io buffer

        :param csv_input:
        :return:
        """
        io_file = io.StringIO(csv_input)
        return next(csv.reader(io_file, delimiter=delimiter))

    def _app(current_tuple, e=None):
        if current_tuple is None or len(current_tuple) == 0:
            return None, "no input"
        csv_list = _csv_to_list(current_tuple)
        if len(csv_list) != len(fields):
            e = {"input": "unexpected number of fields {} obtained {} expected".format(len(csv_list), len(fields))}
            return None, e
        return {k: v for (k, v) in zip(fields, csv_list)}, e
    if fields is None or len(fields) == 0:
        return fixed_input(None, "no fields provided, cannot proceed without order")
    return _app
144866
def read_tsv(cls, file_or_buffer: str):
        """"""
        df = pd.read_csv(file_or_buffer, sep='\t', index_col=0)
        df = df.where(pd.notnull(df), None)
        # Note: df.where(..., None) changes all column types to `object`.
        return cls(df)
345603
def parse_csv_header(line):
    """"""
    units = {}
    names = []
    for var in line.split(','):
        start = var.find('[')
        if start < 0:
            names.append(str(var))
            continue
        else:
            names.append(str(var[:start]))
        end = var.find(']', start)
        unitstr = var[start + 1:end]
        eq = unitstr.find('=')
        if eq >= 0:
            # go past = and ", skip final "
            units[names[-1]] = unitstr[eq + 2:-1]
    return names, units
415063
def _read_csv(filepath, kwargs):
    """"""
    if 'delimiter' not in kwargs:
        kwargs['delimiter'] = ','
    if 'quotechar' not in kwargs:
        kwargs['quotechar'] = '"'
    if 'skiprows' not in kwargs:
        kwargs['skiprows'] = []
    if isinstance(kwargs['skiprows'], int):
        kwargs['skiprows'] = [i for i in range(kwargs['skiprows'])]
    if 'format' in kwargs:
        format_ = kwargs['format']
        kwargs.pop('format', None)
    else:
        format_ = 'default'
    skiprows = kwargs['skiprows']
    kwargs.pop('skiprows', None)

    kwargs_open = {'newline': ''}
    mode = 'r'
    if sys.version_info < (3, 0):
        kwargs_open.pop('newline', None)
        mode = 'rb'
    with open(filepath, mode, **kwargs_open) as fp:
        if format_ == 'default':
            reader = csv.reader(fp, **kwargs)
            data = EList([row for row in reader])
            data = data.remove_indices(skiprows)
        elif format_ == 'dicts':
            reader_list = csv.DictReader(fp, **kwargs)
            data = [row for row in reader_list]
        else:
            raise NotImplementedError('Format \'{}\' unknown'
                                      .format(format_))
    return data
1047830
def _read_input_csv(in_file):
    
    with io.open(in_file, newline=None) as in_handle:
        reader = csv.reader(in_handle)
        next(reader) # header
        for line in reader:
            if line: # empty lines
                (fc_id, lane, sample_id, genome, barcode) = line[:5]
                yield fc_id, lane, sample_id, genome, barcode