Recursive function for parsing hpssacli/ssacli output.
762605
def _get_dict(lines, start_index, indentation, deep):
    """"""

    info = {}
    current_item = None

    i = start_index
    while i < len(lines):

        current_line = lines[i]
        current_line_indentation = _get_indentation(current_line)

        # Check for multi-level returns
        if current_line_indentation < indentation:
            return info, i-1

        if current_line_indentation == indentation:
            current_item = current_line.lstrip(' ')
            info[current_item] = {}
            i = i + 1
            continue

        if i < len(lines) - 1:
            next_line_indentation = _get_indentation(lines[i+1])
        else:
            next_line_indentation = current_line_indentation

        if next_line_indentation > current_line_indentation:
            ret_dict, i = _get_dict(lines, i, current_line_indentation, deep+1)
            for key in ret_dict.keys():
                if key in info[current_item]:
                    info[current_item][key].update(ret_dict[key])
                else:
                    info[current_item][key] = ret_dict[key]
        else:
            key, value = _get_key_value(current_line)
            if key:
                info[current_item][key] = value

        # Do not return if it's the top level of recursion
        if next_line_indentation < current_line_indentation and deep > 0:
            return info, i

        i = i + 1

    return info, i
266347
def parse(handle, seqtype=None, robust=False):
    ''''''
    if seqtype is None:
        seqtype = _get_seqtype_from_ext(handle)

    if seqtype.startswith('gz-'):
        handle = _unzip_handle(handle)
        seqtype = seqtype[3:]

    # False positive from pylint, both handles are fileobj-like
    # pylint: disable=redefined-variable-type
    if robust:
        if seqtype == "embl":
            handle = sanity_check_embl(handle)
        elif seqtype == "genbank":
            handle = sanity_check_genbank(handle)
        elif seqtype == "fasta":
            handle = sanity_check_fasta(handle)
    # pylint: enable=redefined-variable-type

    return SeqIO.parse(handle, seqtype)
1226670
def _parse_header(self, data):
        

        (magic, word_size, byte_order, version, osabi, abi_version, _), data = \
            unpack('4sBBBBB7s', data[:16]), data[16:]

        assert magic == self._ELF_MAGIC, 'Missing ELF magic'
        assert word_size in (1, 2), 'Invalid word size'
        assert byte_order in (1, 2), 'Invalid byte order'

        assert version == 1, 'Invalid version'

        self.osabi = self.OSABI(osabi)
        self.abi_version = abi_version

        endian = Target.Endian(byte_order - 1)
        (type_, machine, version), data = unpack('HHI', data[:8], endian=endian), data[8:]

        try:
            self.type = self.Type(type_)
        except ValueError:
            self.type = self.Type.unknown

        try:
            self.machine = ELF.Machine(machine)
        except ValueError:
            self.machine = ELF.Machine.unknown

        assert version == 1, 'Invalid version'

        if self.machine is ELF.Machine.i386:
            arch = Target.Arch.x86
            assert word_size == 1, 'Unexpected ELF64 for machine type x86'
            assert endian is Target.Endian.little, 'Unexpected big-endian for machine type x86'
        elif self.machine is ELF.Machine.x86_64:
            arch = Target.Arch.x86
            assert word_size == 2, 'Unexpected ELF32 for machine type x64_64'
            assert endian is Target.Endian.little, 'Unexpected big-endian for machine type x86'
        elif self.machine is ELF.Machine.arm:
            arch = Target.Arch.arm
            assert word_size == 1, 'Unexpected ELF64 for machine type arm'
        elif self.machine is ELF.Machine.aarch64:
            arch = Target.Arch.arm
            assert word_size == 2, 'Unexpected ELF32 for machine type aarch64'
        else:
            arch = Target.Arch.unknown

        self.arch = arch
        self.bits = 32 * word_size
        self.endian = endian

        if self.bits == 32:
            fmt = 'IIIIHHHHHH'
        else:
            fmt = 'QQQIHHHHHH'

        fmt_size = pack_size(fmt)
        (self.entry, self.phoff, self.shoff, self.flags, self.hsize, self.phentsize,
            self.phnum, self.shentsize, self.shnum, self.shstrndx) = \
            unpack(fmt, data[:fmt_size], target=self)
229054
def parse_hicpro_stats(self, f, rsection):
        
        s_name = self.clean_s_name(os.path.basename(f['root']), os.path.dirname(f['root']))

        if s_name not in self.hicpro_data.keys():
            self.hicpro_data[s_name] = {}

        self.add_data_source(f, s_name, section=rsection)
        for l in f['f'].splitlines():
            if not l.startswith('#'):
                s = l.split("\t")
                if s[0] in self.hicpro_data[s_name]:
                    log.debug("Duplicated keys found! Overwriting: {}".format(s[0]))
                self.hicpro_data[s_name][s[0]] = int(s[1])
929137
def _parseSegments(self, data, elfHeader):
        """"""
        offset = elfHeader.header.e_phoff
        segments = []
        for i in range(elfHeader.header.e_phnum):
            phdr = self.__classes.PHDR.from_buffer(data, offset)
            segment_bytes = (c_ubyte * phdr.p_filesz).from_buffer(data, phdr.p_offset)

            phdrData = PhdrData(header=phdr, raw=segment_bytes, bytes=bytearray(segment_bytes), type=PT[phdr.p_type], vaddr=phdr.p_vaddr, offset=phdr.p_offset)
            segments.append(phdrData)

            offset += elfHeader.header.e_phentsize

        return segments
688830
def parsecmap(self, fn):
        ''''''
        return self.bind(lambda res: Parser(lambda _, index: Value.success(index, fn(res))))
1296791
def analyze_fa(fa):
    
    if fa.name == '<stdin>':
        safe = 'temp.id'
    else:
        safe = '%s.id' % (fa.name)
    safe = open(safe, 'w')
    sequences = {} # sequences[id] = sequence
    insertions = {} # insertions[id] = [[start, stop], [start, stop], ...]
    count = 0
    id2name = {}
    names = []
    for seq in parse_fasta(fa):
        id = '%010d' % (count,)
        name = seq[0].split('>', 1)[1]
        id2name[id] = name
        id2name[name] = id
        names.append(name)
        insertions[id] = insertions_from_masked(seq[1])
        sequences[id] = seq
        print('\n'.join(['>%s' % (id), seq[1].upper()]), file=safe)
        count += 1
    safe.close()
    lookup = open('%s.id.lookup' % (fa.name), 'w')
    for i in list(id2name.items()):
        print('\t'.join(i), file=lookup)
    lookup.close()
    return safe.name, sequences, id2name, names, insertions
394352
def parse_header(filename):
    '''
    '''
    with open(filename, 'rb') as fh:
        content = '\n'.join(fh.read().splitlines())

    content = sub('\t', ' ', content)
    content = strip_comments(content)

    # first get the functions
    content = split(func_pat_short, content)

    for i, s in enumerate(content):
        if i % 2 and content[i].strip():  # matched a prototype
            try:
                content[i] = parse_prototype(content[i])
            except Exception as e:
                traceback.print_exc()

    # now process structs
    res = []
    for i, item in enumerate(content):
        if not isinstance(item, str):  # if it's already a func etc. skip it
            res.append(item)
            continue

        items = split(struct_pat, item)
        j = 0
        while j < len(items):
            if not j % 5:
                res.append(items[j])
                j += 1
            else:
                if items[j].strip() == 'enum':
                    res.append(parse_enum(*items[j + 1: j + 4]))
                else:
                    res.append(parse_struct(*items[j + 1: j + 4]))
                j += 4

    # now do remaining simple typedefs
    content = res
    res = []
    for i, item in enumerate(content):
        if not isinstance(item, str):  # if it's already processed skip it
            res.append(item)
            continue

        items = split(typedef_pat, item)
        for j, item in enumerate(items):
            res.append(TypeDef(item.strip()) if j % 2 else item)

    content = [c for c in res if not isinstance(c, str) or c.strip()]
    return content
141131
def parse (cls, line, lineno, log, cmddict=None):
    
    start  = line.find('{')
    stop   = line.find('}')
    pos    = SeqPos(line, lineno, start + 1, stop)
    result = cls(None, pos)

    if start >= 0 and stop >= start:
      attrs = { }
      pairs = line[start + 1:stop].split(',')

      for item in pairs:
        ncolons = item.count(':')
        if ncolons == 0:
          log.error('Missing colon in command attribute "%s".' % item, pos)
        elif ncolons > 1:
          log.error('Too many colons in command attribute "%s".' % item, pos)
        else:
          name, value = (s.strip() for s in item.split(':'))
          attrs[name] = value

      result = cls(attrs, pos)

    elif start != -1 or stop != -1:
      log.error('Incorrect command attribute curly brace placement.', pos)

    return result
911569
def _convert_parsed_show_output(parsed_data):
    '''
    
    '''
    # Match lines like "main: xenial [snapshot]" or "test [local]".
    source_pattern = re.compile(r'(?:(?P<component>\S+):)?\s*(?P<name>\S+)\s+\[(?P<type>\S+)\]')
    sources = list()

    if 'architectures' in parsed_data:
        parsed_data['architectures'] = [item.strip() for item in parsed_data['architectures'].split()]
        parsed_data['architectures'] = sorted(parsed_data['architectures'])

    for source in parsed_data.get('sources', []):
        # Retain the key/value of only the matching named groups.
        matches = source_pattern.search(source)

        if matches:
            groups = matches.groupdict()
            sources.append({key: groups[key] for key in groups if groups[key]})
    if sources:
        parsed_data['sources'] = sources

    return parsed_data
750573
def _parse_parallel_sentences(f1, f2):
  """"""
  def _parse_text(path):
    """Returns the sentences from a single text file, which may be gzipped."""
    split_path = path.split(".")

    if split_path[-1] == "gz":
      lang = split_path[-2]
      with tf.io.gfile.GFile(path) as f, gzip.GzipFile(fileobj=f) as g:
        return g.read().split("\n"), lang

    if split_path[-1] == "txt":
      # CWMT
      lang = split_path[-2].split("_")[-1]
      lang = "zh" if lang in ("ch", "cn") else lang
    else:
      lang = split_path[-1]
    with tf.io.gfile.GFile(path) as f:
      return f.read().split("\n"), lang

  def _parse_sgm(path):
    """Returns sentences from a single SGML file."""
    lang = path.split(".")[-2]
    sentences = []
    # Note: We can't use the XML parser since some of the files are badly
    # formatted.
    seg_re = re.compile(r"<seg id=\"\d+\">(.*)</seg>")
    with tf.io.gfile.GFile(path) as f:
      for line in f:
        seg_match = re.match(seg_re, line)
        if seg_match:
          assert len(seg_match.groups()) == 1
          sentences.append(seg_match.groups()[0])
    return sentences, lang

  parse_file = _parse_sgm if f1.endswith(".sgm") else _parse_text

  # Some datasets (e.g., CWMT) contain multiple parallel files specified with
  # a wildcard. We sort both sets to align them and parse them one by one.
  f1_files = tf.io.gfile.glob(f1)
  f2_files = tf.io.gfile.glob(f2)

  assert f1_files and f2_files, "No matching files found: %s, %s." % (f1, f2)
  assert len(f1_files) == len(f2_files), (
      "Number of files do not match: %d vs %d for %s vs %s." % (
          len(f1_files), len(f2_files), f1, f2))

  for f1_i, f2_i in zip(sorted(f1_files), sorted(f2_files)):
    l1_sentences, l1 = parse_file(f1_i)
    l2_sentences, l2 = parse_file(f2_i)

    assert len(l1_sentences) == len(l2_sentences), (
        "Sizes do not match: %d vs %d for %s vs %s." % (
            len(l1_sentences), len(l2_sentences), f1_i, f2_i))

    for s1, s2 in zip(l1_sentences, l2_sentences):
      yield {
          l1: s1,
          l2: s2
      }
90318
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
            knowledge_base):
    """"""
    _ = stderr, time_taken, args, knowledge_base  # Unused
    self.CheckReturn(cmd, return_val)

    plist = biplist.readPlist(io.BytesIO(stdout))

    if len(plist) > 1:
      raise parser.ParseError("SPHardwareDataType plist has too many items.")

    hardware_list = plist[0]["_items"][0]
    serial_number = hardware_list.get("serial_number", None)
    system_product_name = hardware_list.get("machine_model", None)
    bios_version = hardware_list.get("boot_rom_version", None)

    yield rdf_client.HardwareInfo(
        serial_number=serial_number,
        bios_version=bios_version,
        system_product_name=system_product_name)
1062372
def parse_fasta_annotations(fastas, annot_tables, trans_table):
    
    if annot_tables is not False:
        annots = {}
        for table in annot_tables:
            for cds in open(table):
                ID, start, end, strand = cds.strip().split()
                annots[ID] = [start, end, int(strand)]
    for fasta in fastas:
        for seq in parse_fasta(fasta):
            if ('# ;gc_cont' not in seq[0] and '# ID=' not in seq[0]) and annot_tables is False:
                print('# specify fasta from Prodigal or annotations table (-t)', file=sys.stderr)
                exit()
            if 'ID=' in seq[0]:
                ID = seq[0].rsplit('ID=', 1)[1].split(';', 1)[0]
                contig = seq[0].split()[0].split('>')[1].rsplit('_%s' % (ID), 1)[0]
            else:
                contig = seq[0].split()[0].split('>')[1].rsplit('_', 1)[0]
            locus = seq[0].split()[0].split('>')[1]
            # annotation info from Prodigal
            if ('# ;gc_cont' in seq[0] or '# ID=' in seq[0]):
                info = seq[0].split(' # ')
                start, end, strand = int(info[1]), int(info[2]), info[3]
                if strand == '1':
                    strand = 1
                else:
                    strand = -1
                product = [''.join(info[4].split()[1:])]
            # annotation info from table
            else:
                start, end, strand = annots[locus]
                product = seq[0].split(' ', 1)[1]
            info = {'transl_table':[trans_table], \
                    'translation':[seq[1]], \
                    'product':product}
            yield contig, [locus, [start, end, strand], info]
321482
def _parse_headers(headers):
    ""
    try:
        return dict(line.rstrip().split(": ", 1)
            for line in headers.splitlines()
            if line
        )
    except (TypeError, ValueError) as exc:
        raise SCGIException("Error in SCGI headers %r (%s)" % (headers, exc,))
393438
def _parse_proc_cgroups(self):
        """
        #subsys_name	hierarchy	num_cgroups	enabled
        cpuset	0	1	1
        ns	0	1	1
        cpu	1	10	1
        cpuacct	0	1	1
        memory	0	1	1
        devices	0	1	1
        freezer	0	1	1
        net_cls	0	1	1
        """
        for line in fileops.readlines('/proc/cgroups'):
            m = self._RE_CGROUPS.match(line)
            if m is None:
                continue

            name = m.group('name')
            hierarchy = int(m.group('hier'))
            n_cgroups = int(m.group('n'))
            if m.group('enabled') == '1':
                enabled = True
            else:
                enabled = False
            if name not in self:
                self[name] = {}
            self[name]['name'] = name
            self[name]['hierarchy'] = hierarchy
            self[name]['num_cgroups'] = n_cgroups
            self[name]['enabled'] = enabled
206392
def _glsa_list_process_output(output):
    '''
    
    '''
    ret = dict()
    for line in output:
        try:
            glsa_id, status, desc = line.split(None, 2)
            if 'U' in status:
                status += ' Not Affected'
            elif 'N' in status:
                status += ' Might be Affected'
            elif 'A' in status:
                status += ' Applied (injected)'
            if 'CVE' in desc:
                desc, cves = desc.rsplit(None, 1)
                cves = cves.split(',')
            else:
                cves = list()
            ret[glsa_id] = {'description': desc, 'status': status,
                            'CVEs': cves}
        except ValueError:
            pass
    return ret
691805
def parse_fasta(infile, upper=False):
    '''
    
    '''
    try:
        fp = must_open(infile)
    except:
        fp = infile
    # keep header
    fa_iter = (x[1] for x in groupby(fp, lambda row: row[0] == '>'))
    for header in fa_iter:
        header = next(header)
        if header[0] != '>':
            continue
        # drop '>'
        header = header.strip()[1:]
        # stitch the sequence lines together and make into upper case
        seq = "".join(s.strip() for s in next(fa_iter))
        if upper:
            seq = seq.upper()
        yield header, seq